Miscellaneous improvements

- Upgrade to BS3 final.
- Improve crawling.
- Protect GitHub API quota from unfriendly robots.
- Other small fixes.

database/database.go

@@ -170,12 +170,6 @@ var putScript = redis.NewScript(0, `
             redis.call('SREM', 'index:' .. term, id)
         elseif x == 2 then 
             redis.call('SADD', 'index:' .. term, id)
-            if string.sub(term, 1, 7) == 'import:' then
-                local import = string.sub(term, 8)
-                if redis.call('HEXISTS', 'ids',  import) == 0  and redis.call('SISMEMBER', 'badCrawl', import) == 0 then
-                    redis.call('SADD', 'newCrawl', import)
-                end
-            end
         end
     end
 
@@ -190,6 +184,15 @@ var putScript = redis.NewScript(0, `
     return redis.call('HMSET', 'pkg:' .. id, 'path', path, 'synopsis', synopsis, 'score', score, 'gob', gob, 'terms', terms, 'etag', etag, 'kind', kind)
 `)
 
+var addCrawlScript = redis.NewScript(0, `
+    for i=1,#ARGV do
+        local pkg = ARGV[i]
+        if redis.call('HEXISTS', 'ids',  pkg) == 0  and redis.call('SISMEMBER', 'badCrawl', pkg) == 0 then
+            redis.call('SADD', 'newCrawl', pkg)
+        end
+    end
+`)
+
 // Put adds the package documentation to the database.
 func (db *Database) Put(pdoc *doc.Package, nextCrawl time.Time) error {
 	c := db.Pool.Get()
@@ -204,7 +207,7 @@ func (db *Database) Put(pdoc *doc.Package, nextCrawl time.Time) error {
 	}
 
 	// Truncate large documents.
-	if gobBuf.Len() > 700000 {
+	if gobBuf.Len() > 200000 {
 		pdocNew := *pdoc
 		pdoc = &pdocNew
 		pdoc.Truncated = true
@@ -236,7 +239,45 @@ func (db *Database) Put(pdoc *doc.Package, nextCrawl time.Time) error {
 	if !nextCrawl.IsZero() {
 		t = nextCrawl.Unix()
 	}
+
 	_, err = putScript.Do(c, pdoc.ImportPath, pdoc.Synopsis, score, gobBytes, strings.Join(terms, " "), pdoc.Etag, kind, t)
+	if err != nil {
+		return err
+	}
+
+	if nextCrawl.IsZero() {
+		// Skip crawling related packages if this is not a full save.
+		return nil
+	}
+
+	paths := make(map[string]bool)
+	for _, p := range pdoc.Imports {
+		if doc.IsValidRemotePath(p) {
+			paths[p] = true
+		}
+	}
+	for _, p := range pdoc.TestImports {
+		if doc.IsValidRemotePath(p) {
+			paths[p] = true
+		}
+	}
+	for _, p := range pdoc.XTestImports {
+		if doc.IsValidRemotePath(p) {
+			paths[p] = true
+		}
+	}
+	if pdoc.ImportPath != pdoc.ProjectRoot && pdoc.ProjectRoot != "" {
+		paths[pdoc.ProjectRoot] = true
+	}
+	for _, p := range pdoc.Subdirectories {
+		paths[pdoc.ImportPath+"/"+p] = true
+	}
+
+	args := make([]interface{}, 0, len(paths))
+	for p := range paths {
+		args = append(args, p)
+	}
+	_, err = addCrawlScript.Do(c, args...)
 	return err
 }
 
@@ -456,7 +497,6 @@ var deleteScript = redis.NewScript(0, `
     end
 
     redis.call('ZREM', 'nextCrawl', id)
-    redis.call('SREM', 'badCrawl', path)
     redis.call('SREM', 'newCrawl', path)
     redis.call('ZREM', 'popular', id)
     redis.call('DEL', 'pkg:' .. id)
@@ -670,6 +710,7 @@ type PackageInfo struct {
 	Pkgs  []Package
 	Score float64
 	Kind  string
+	Size  int
 }
 
 // Do executes function f for each document in the database.
@@ -681,7 +722,7 @@ func (db *Database) Do(f func(*PackageInfo) error) error {
 		return err
 	}
 	for _, key := range keys {
-		values, err := redis.Values(c.Do("HMGET", key, "gob", "score", "kind", "path"))
+		values, err := redis.Values(c.Do("HMGET", key, "gob", "score", "kind", "path", "terms", "synopis"))
 		if err != nil {
 			return err
 		}
@@ -690,9 +731,11 @@ func (db *Database) Do(f func(*PackageInfo) error) error {
 			pi       PackageInfo
 			p        []byte
 			path     string
+			terms    string
+			synopsis string
 		)
 
-		if _, err := redis.Scan(values, &p, &pi.Score, &pi.Kind, &path); err != nil {
+		if _, err := redis.Scan(values, &p, &pi.Score, &pi.Kind, &path, &terms, &synopsis); err != nil {
 			return err
 		}
 
@@ -700,6 +743,8 @@ func (db *Database) Do(f func(*PackageInfo) error) error {
 			continue
 		}
 
+		pi.Size = len(path) + len(p) + len(terms) + len(synopsis)
+
 		p, err = snappy.Decode(nil, p)
 		if err != nil {
 			return fmt.Errorf("snappy decoding %s: %v", path, err)
@@ -810,7 +855,7 @@ func (db *Database) GetGob(key string, value interface{}) error {
 	return gob.NewDecoder(bytes.NewReader(p)).Decode(value)
 }
 
-var incrementPopularScore = redis.NewScript(0, `
+var incrementPopularScoreScript = redis.NewScript(0, `
     local path = ARGV[1]
     local n = ARGV[2]
     local t = ARGV[3]
@@ -832,20 +877,21 @@ var incrementPopularScore = redis.NewScript(0, `
 
 const popularHalfLife = time.Hour * 24 * 7
 
-func scaledTime(t time.Time) float64 {
+func (db *Database) incrementPopularScoreInternal(path string, delta float64, t time.Time) error {
-	const lambda = math.Ln2 / float64(popularHalfLife)
-	return lambda * float64(t.Sub(time.Unix(1257894000, 0)))
-}
-
-func (db *Database) IncrementPopularScore(path string) error {
 	// nt = n0 * math.Exp(-lambda * t)
 	// lambda = math.Ln2 / thalf
 	c := db.Pool.Get()
 	defer c.Close()
-	_, err := incrementPopularScore.Do(c, path, 1, scaledTime(time.Now()))
+	const lambda = math.Ln2 / float64(popularHalfLife)
+	scaledTime := lambda * float64(t.Sub(time.Unix(1257894000, 0)))
+	_, err := incrementPopularScoreScript.Do(c, path, delta, scaledTime)
 	return err
 }
 
+func (db *Database) IncrementPopularScore(path string) error {
+	return db.incrementPopularScoreInternal(path, 1, time.Now())
+}
+
 var popularScript = redis.NewScript(0, `
     local stop = ARGV[1]
     local ids = redis.call('ZREVRANGE', 'popular', '0', stop)
@@ -892,26 +938,59 @@ func (db *Database) PopularWithScores() ([]Package, error) {
 	return pkgs, err
 }
 
-func (db *Database) GetNewCrawl() (string, error) {
+func (db *Database) PopNewCrawl() (string, bool, error) {
 	c := db.Pool.Get()
 	defer c.Close()
-	v, err := redis.String(c.Do("SRANDMEMBER", "newCrawl"))
+
-	if err == redis.ErrNil {
+	var subdirs []Package
+
+	path, err := redis.String(c.Do("SPOP", "newCrawl"))
+	switch {
+	case err == redis.ErrNil:
 		err = nil
+		path = ""
+	case err == nil:
+		subdirs, err = db.getSubdirs(c, path, nil)
 	}
-	return v, err
+	return path, len(subdirs) > 0, err
 }
 
-var setBadCrawlScript = redis.NewScript(0, `
+func (db *Database) AddBadCrawl(path string) error {
-    local path = ARGV[1]
-    if redis.call('SREM', 'newCrawl', path) == 1 then
-        redis.call('SADD', 'badCrawl', path)
-    end
-`)
-
-func (db *Database) SetBadCrawl(path string) error {
 	c := db.Pool.Get()
 	defer c.Close()
-	_, err := setBadCrawlScript.Do(c, path)
+	_, err := c.Do("SADD", "badCrawl", path)
 	return err
 }
+
+var incrementCounterScript = redis.NewScript(0, `
+    local key = 'counter:' .. ARGV[1]
+    local n = tonumber(ARGV[2])
+    local t = tonumber(ARGV[3])
+    local exp = tonumber(ARGV[4])
+
+    local counter = redis.call('GET', key)
+    if counter then
+        counter = cjson.decode(counter)
+        n = n + counter.n * math.exp(counter.t - t)
+    end
+
+    redis.call('SET', key, cjson.encode({n = n; t = t}))
+    redis.call('EXPIRE', key, exp)
+    return tostring(n)
+`)
+
+const counterHalflife = time.Hour
+
+func (db *Database) incrementCounterInternal(key string, delta float64, t time.Time) (float64, error) {
+	// nt = n0 * math.Exp(-lambda * t)
+	// lambda = math.Ln2 / thalf
+	c := db.Pool.Get()
+	defer c.Close()
+	const lambda = math.Ln2 / float64(counterHalflife)
+	scaledTime := lambda * float64(t.Sub(time.Unix(1257894000, 0)))
+	return redis.Float64(incrementCounterScript.Do(c, key, delta, scaledTime, (4*counterHalflife)/time.Second))
+}
+
+func (db *Database) IncrementCounter(key string, delta float64) (float64, error) {
+	return db.incrementCounterInternal(key, delta, time.Now())
+}

database/database_test.go

@@ -193,6 +193,8 @@ func TestPutGet(t *testing.T) {
 	}
 }
 
+const epsilon = 0.000001
+
 func TestPopular(t *testing.T) {
 	db := newDB(t)
 	defer closeDB(db)
@@ -207,7 +209,7 @@ func TestPopular(t *testing.T) {
 	for id := 12; id >= 0; id-- {
 		path := "github.com/user/repo/p" + strconv.Itoa(id)
 		c.Do("HSET", "ids", path, id)
-		_, err := incrementPopularScore.Do(c, path, score, scaledTime(now))
+		err := db.incrementPopularScoreInternal(path, score, now)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -227,8 +229,39 @@ func TestPopular(t *testing.T) {
 	}
 	for i := 3; i < len(values); i += 2 {
 		s, _ := redis.Float64(values[i], nil)
-		if math.Abs(score-s)/score > 0.0001 {
+		if math.Abs(score-s)/score > epsilon {
 			t.Errorf("Bad score, score[1]=%g, score[%d]=%g", score, i, s)
 		}
 	}
 }
+
+func TestCounter(t *testing.T) {
+	db := newDB(t)
+	defer closeDB(db)
+
+	const key = "127.0.0.1"
+
+	now := time.Now()
+	n, err := db.incrementCounterInternal(key, 1, now)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if math.Abs(n-1.0) > epsilon {
+		t.Errorf("1: got n=%g, want 1", n)
+	}
+	n, err = db.incrementCounterInternal(key, 1, now)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if math.Abs(n-2.0)/2.0 > epsilon {
+		t.Errorf("2: got n=%g, want 2", n)
+	}
+	now = now.Add(counterHalflife)
+	n, err = db.incrementCounterInternal(key, 1, now)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if math.Abs(n-2.0)/2.0 > epsilon {
+		t.Errorf("3: got n=%g, want 2", n)
+	}
+}

doc/bitbucket.go

@@ -63,18 +63,19 @@ func getBitbucketDoc(client *http.Client, match map[string]string, savedEtag str
 		return nil, ErrNotModified
 	}
 
-	var directory struct {
+	var contents struct {
+		Directories []string
 		Files       []struct {
 			Path string
 		}
 	}
 
-	if err := httpGetJSON(client, expand("https://api.bitbucket.org/1.0/repositories/{owner}/{repo}/src/{tag}{dir}/", match), nil, &directory); err != nil {
+	if err := httpGetJSON(client, expand("https://api.bitbucket.org/1.0/repositories/{owner}/{repo}/src/{tag}{dir}/", match), nil, &contents); err != nil {
 		return nil, err
 	}
 
 	var files []*source
-	for _, f := range directory.Files {
+	for _, f := range contents.Files {
 		_, name := path.Split(f.Path)
 		if isDocFile(name) {
 			files = append(files, &source{
@@ -99,6 +100,7 @@ func getBitbucketDoc(client *http.Client, match map[string]string, savedEtag str
 			BrowseURL:      expand("https://bitbucket.org/{owner}/{repo}/src/{tag}{dir}", match),
 			Etag:           etag,
 			VCS:            match["vcs"],
+			Subdirectories: contents.Directories,
 		},
 	}
 

doc/builder.go

@@ -16,6 +16,7 @@ package doc
 
 import (
 	"bytes"
+	"errors"
 	"go/ast"
 	"go/build"
 	"go/doc"
@@ -327,7 +328,10 @@ var packageNamePats = []*regexp.Regexp{
 
 func simpleImporter(imports map[string]*ast.Object, path string) (*ast.Object, error) {
 	pkg := imports[path]
-	if pkg == nil {
+	if pkg != nil {
+		return pkg, nil
+	}
+
 	// Guess the package name without importing it.
 	for _, pat := range packageNamePats {
 		m := pat.FindStringSubmatch(path)
@@ -335,12 +339,12 @@ func simpleImporter(imports map[string]*ast.Object, path string) (*ast.Object, e
 			pkg = ast.NewObj(ast.Pkg, m[1])
 			pkg.Data = ast.NewScope(nil)
 			imports[path] = pkg
-				break
-			}
-		}
-	}
 			return pkg, nil
 		}
+	}
+
+	return nil, errors.New("package not found")
+}
 
 type File struct {
 	Name string

doc/github.go

@@ -17,7 +17,6 @@ package doc
 import (
 	"net/http"
 	"net/url"
-	"path"
 	"regexp"
 	"strings"
 	"time"
@@ -73,6 +72,47 @@ func getGitHubDoc(client *http.Client, match map[string]string, savedEtag string
 		return nil, ErrNotModified
 	}
 
+	var contents []*struct {
+		Type     string
+		Name     string
+		Git_URL  string
+		HTML_URL string
+	}
+
+	err = httpGetJSON(client, expand("https://api.github.com/repos/{owner}/{repo}/contents{dir}?ref={tag}&{cred}", match), nil, &contents)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(contents) == 0 {
+		return nil, NotFoundError{"No files in directory."}
+	}
+
+	// Because Github API URLs are case-insensitive, we check that the owner
+	// and repo returned from Github matches the one that we are requesting.
+	if !strings.HasPrefix(contents[0].Git_URL, expand("https://api.github.com/repos/{owner}/{repo}/", match)) {
+		return nil, NotFoundError{"Github import path has incorrect case."}
+	}
+
+	var files []*source
+	var subdirs []string
+
+	for _, item := range contents {
+		switch {
+		case item.Type == "dir":
+			if isValidPathElement(item.Name) {
+				subdirs = append(subdirs, item.Name)
+			}
+		case isDocFile(item.Name):
+			files = append(files, &source{
+				name:      item.Name,
+				browseURL: item.HTML_URL,
+				rawURL:    item.Git_URL + "?" + gitHubCred,
+			})
+		}
+	}
+
+	/*
 		var tree struct {
 			Tree []struct {
 				Url  string
@@ -116,6 +156,7 @@ func getGitHubDoc(client *http.Client, match map[string]string, savedEtag string
 		if !inTree {
 			return nil, NotFoundError{"Directory tree does not contain Go files."}
 		}
+	*/
 
 	if err := fetchFiles(client, files, gitHubRawHeader); err != nil {
 		return nil, err
@@ -136,6 +177,7 @@ func getGitHubDoc(client *http.Client, match map[string]string, savedEtag string
 			BrowseURL:      browseURL,
 			Etag:           commit,
 			VCS:            "git",
+			Subdirectories: subdirs,
 		},
 	}
 

doc/google.go

@@ -26,7 +26,7 @@ var (
 	googleRepoRe     = regexp.MustCompile(`id="checkoutcmd">(hg|git|svn)`)
 	googleRevisionRe = regexp.MustCompile(`<h2>(?:[^ ]+ - )?Revision *([^:]+):`)
 	googleEtagRe     = regexp.MustCompile(`^(hg|git|svn)-`)
-	googleFileRe     = regexp.MustCompile(`<li><a href="([^"/]+)"`)
+	googleFileRe     = regexp.MustCompile(`<li><a href="([^"]+)"`)
 	googlePattern    = regexp.MustCompile(`^code\.google\.com/p/(?P<repo>[a-z0-9\-]+)(:?\.(?P<subrepo>[a-z0-9\-]+))?(?P<dir>/[a-z0-9A-Z_.\-/]+)?$`)
 )
 
@@ -54,10 +54,17 @@ func getGoogleDoc(client *http.Client, match map[string]string, savedEtag string
 		}
 	}
 
+	var subdirs []string
 	var files []*source
 	for _, m := range googleFileRe.FindAllSubmatch(p, -1) {
 		fname := string(m[1])
-		if isDocFile(fname) {
+		switch {
+		case strings.HasSuffix(fname, "/"):
+			fname = fname[:len(fname)-1]
+			if isValidPathElement(fname) {
+				subdirs = append(subdirs, fname)
+			}
+		case isDocFile(fname):
 			files = append(files, &source{
 				name:      fname,
 				browseURL: expand("http://code.google.com/p/{repo}/source/browse{dir}/{0}{query}", match, fname),

doc/path.go

@@ -487,6 +487,10 @@ var validTLD = map[string]bool{
 var validHost = regexp.MustCompile(`^[-a-z0-9]+(?:\.[-a-z0-9]+)+$`)
 var validPathElement = regexp.MustCompile(`^[-A-Za-z0-9~+][-A-Za-z0-9_.]*$`)
 
+func isValidPathElement(s string) bool {
+	return validPathElement.MatchString(s) && s != "testdata"
+}
+
 // IsValidRemotePath returns true if importPath is structurally valid for "go get".
 func IsValidRemotePath(importPath string) bool {
 
@@ -506,7 +510,7 @@ func IsValidRemotePath(importPath string) bool {
 	}
 
 	for _, part := range parts[1:] {
-		if !validPathElement.MatchString(part) || part == "testdata" {
+		if !isValidPathElement(part) {
 			return false
 		}
 	}

doc/vcs.go

@@ -89,7 +89,7 @@ type vcsCmd struct {
 }
 
 var vcsCmds = map[string]*vcsCmd{
-	"git": &vcsCmd{
+	"git": {
 		schemes:  []string{"http", "https", "git"},
 		download: downloadGit,
 	},
@@ -216,10 +216,14 @@ func getVCSDoc(client *http.Client, match map[string]string, etagSaved string) (
 	}
 
 	var files []*source
+	var subdirs []string
 	for _, fi := range fis {
-		if fi.IsDir() || !isDocFile(fi.Name()) {
+		switch {
-			continue
+		case fi.IsDir():
+			if isValidPathElement(fi.Name()) {
+				subdirs = append(subdirs, fi.Name())
 			}
+		case isDocFile(fi.Name()):
 			b, err := ioutil.ReadFile(path.Join(d, fi.Name()))
 			if err != nil {
 				return nil, err
@@ -230,6 +234,7 @@ func getVCSDoc(client *http.Client, match map[string]string, etagSaved string) (
 				data:      b,
 			})
 		}
+	}
 
 	// Create the documentation.
 
@@ -243,6 +248,7 @@ func getVCSDoc(client *http.Client, match map[string]string, etagSaved string) (
 			BrowseURL:      "",
 			Etag:           etag,
 			VCS:            match["vcs"],
+			Subdirectories: subdirs,
 		},
 	}
 

gddo-admin/stats.go

@@ -15,8 +15,10 @@
 package main
 
 import (
+	"fmt"
 	"log"
 	"os"
+	"sort"
 
 	"github.com/garyburd/gddo/database"
 )
@@ -27,14 +29,47 @@ var statsCommand = &command{
 	usage: "stats",
 }
 
+type itemSize struct {
+	path string
+	size int
+}
+
+type bySizeDesc []itemSize
+
+func (p bySizeDesc) Len() int           { return len(p) }
+func (p bySizeDesc) Less(i, j int) bool { return p[i].size > p[j].size }
+func (p bySizeDesc) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+
 func stats(c *command) {
 	if len(c.flag.Args()) != 0 {
 		c.printUsage()
 		os.Exit(1)
 	}
-	_, err := database.New()
+	db, err := database.New()
 	if err != nil {
 		log.Fatal(err)
 	}
-	log.Println("DONE")
+
+	var packageSizes []itemSize
+	projectSizes := make(map[string]int)
+	err = db.Do(func(pi *database.PackageInfo) error {
+		packageSizes = append(packageSizes, itemSize{pi.PDoc.ImportPath, pi.Size})
+		projectSizes[pi.PDoc.ProjectRoot] += pi.Size
+		return nil
+	})
+
+	var sizes []itemSize
+	for path, size := range projectSizes {
+		sizes = append(sizes, itemSize{path, size})
+	}
+	sort.Sort(bySizeDesc(sizes))
+	for _, size := range sizes {
+		fmt.Printf("%6d %s\n", size.size, size.path)
+	}
+
+	sort.Sort(bySizeDesc(packageSizes))
+	for _, size := range packageSizes {
+		fmt.Printf("%6d %s\n", size.size, size.path)
+	}
+
 }

gddo-server/assets/site.css

@@ -12,10 +12,13 @@ h4 { margin-top: 20px; }
 }
 
 #x-footer {
-    padding-top: 15px;
+    padding-top: 14px;
     padding-bottom: 15px;
     margin-top: 5px;
     background-color: #eee;
+    border-top-style: solid;
+    border-top-width: 1px;
+
 }
 
 #x-pkginfo {
@@ -25,12 +28,6 @@ h4 { margin-top: 20px; }
     margin-bottom: 15px;
 }
 
-@media screen and (min-width: 768px) {
-    #x-search {
-        float: right;
-    }
-}
-  
 code { 
   background-color: inherit; 
   border: none; 
@@ -55,22 +52,29 @@ pre .com {
   color: rgb(147, 161, 161);
 }
 
-a, .navbar-brand {
+a, .navbar-default .navbar-brand {
     color: #375eab;
 }
 
-.btn-default {
+.navbar-default, #x-footer {
-    background-color: #375eab;
+    background-color: hsl(209, 51%, 92%);
+    border-color: hsl(209, 51%, 88%);
 }
 
-.navbar, #x-footer {
+.navbar-default .navbar-nav > .active > a,
-    background-color: #E0EBF5;
+.navbar-default .navbar-nav > .active > a:hover,
+.navbar-default .navbar-nav > .active > a:focus {
+    background-color: hsl(209, 51%, 88%);
 }
 
-.navbar-nav > .active > a,
+.navbar-default .navbar-nav > li > a:hover,
-.navbar-nav > .active > a:hover,
+.navbar-default .navbar-nav > li > a:focus {
-.navbar-nav > .active > a:focus {
+    color: #000;
-  background-color: #e7e7e7;
+}
+
+.panel-default > .panel-heading {
+    color: #333;
+    background-color: transparent;
 }
 
 #x-file .highlight {

gddo-server/assets/site.js

@@ -72,8 +72,10 @@ $(function() {
 
     $('span.timeago').timeago();
     if (window.location.hash.substring(0, 9) == '#example-') {
-        console.log(window.location.hash.substring(1, 9));
+        var id = '#ex-' + window.location.hash.substring(9);
-       $('#ex-' + window.location.hash.substring(9)).addClass('in').height('auto');
+        console.log(id);
+        console.log($(id));
+       $(id).addClass('in').removeClass('collapse').height('auto');
     }
 
     var highlighted;

gddo-server/assets/templates/home.html

@@ -15,7 +15,7 @@ and more.
 <div class="row">
   <div class="col-sm-6">
     {{with .Popular}}
-      <h4>Popular Packages</h4>
+      <h4>Popular Projects</h4>
       <ul class="list-unstyled">
         {{range .}}<li><a href="/{{.Path}}">{{.Path}}</a>{{end}}
       </ul>
@@ -28,6 +28,7 @@ and more.
       <li><a href="/-/go">Go Standard Packages</a>
       <li><a href="/-/subrepo">Go Sub-repository Packages</a>
       <li><a href="https://code.google.com/p/go-wiki/wiki/Projects">Projects @ go-wiki</a>
+      <li><a href="http://go-search.org/">Go Search</a>
     </ul>
   </div>
 </div>

gddo-server/assets/templates/layout.html

@@ -6,34 +6,38 @@
   {{template "Head" $}}
 </head>
 <body>
-<div class="navbar navbar-static-top">
+<nav class="navbar navbar-default" role="navigation">
   <div class="container">
-    <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
+  <div class="navbar-header">
+    <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
+      <span class="sr-only">Toggle navigation</span>
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="/">GoDoc</a>
+    <a class="navbar-brand" href="/"><strong>GoDoc</strong></a>
-    <div class="nav-collapse collapse navbar-responsive-collapse">
+  </div>
+  <div class="collapse navbar-collapse">
     <ul class="nav navbar-nav">
         <li{{if equal "home.html" templateName}} class="active"{{end}}><a href="/">Home</a></li>
         <li{{if equal "index.html" templateName}} class="active"{{end}}><a href="/-/index">Index</a></li>
         <li{{if equal "about.html" templateName}} class="active"{{end}}><a href="/-/about">About</a></li>
     </ul>
-      <form class="navbar-form" id="x-search" action="/"><input class="form-control" id="x-search-query" type="text" name="q" placeholder="Search"></form>
+    <form class="navbar-nav navbar-form navbar-right" id="x-search" action="/" role="search"><input class="form-control" id="x-search-query" type="text" name="q" placeholder="Search"></form>
   </div>
 </div>
-</div>
+</nav>
-<div class="container"><div class="row"><div class="col-12">
+
+<div class="container">
   {{template "Body" $}}
-</div></div></div>
+</div>
 <div id="x-footer" class="clearfix">
-  <div class="container"><div class="row"><div class="col-12">
+  <div class="container">
     <a href="mailto:info@godoc.org">Feedback</a>
     <span class="text-muted">|</span> <a href="https://github.com/garyburd/gddo/issues">Website Issues</a>
     <span class="text-muted">|</span> <a href="http://golang.org/">Go Language</a>
     <span class="pull-right"><a href="#">Back to top</a></span>
-  </div></div></div>
+  </div>
 </div>
 
 <div id="x-shortcuts" tabindex="-1" class="modal fade">

gddo-server/assets/templates/pkg.html

@@ -25,7 +25,7 @@
     </ul>
 
     {{with .AllExamples}}<h4 id="pkg-examples">Examples <a class="permalink" href="#pkg-examples">&para;</a></h4><ul class="list-unstyled">{{range . }}
-    <li><a href="#example-{{.Id}}" onclick="$('#ex-{{.Id}}').addClass('in').height('auto')">{{.Label}}</a>{{end}}
+    <li><a href="#example-{{.Id}}" onclick="$('#ex-{{.Id}}').addClass('in').removeClass('collapse').height('auto')">{{.Label}}</a>{{end}}
     </ul>{{else}}<span id="pkg-examples"></span>{{end}}
 
     <h4 id="pkg-files">{{with .BrowseURL}}<a href="{{.}}">Package Files</a>{{else}}Package Files{{end}} <a class="permalink" href="#pkg-files">&para;</a></h4>
@@ -65,10 +65,10 @@
 {{template "PkgCmdFooter" $}}
 {{end}}
 
-{{define "Examples"}}{{if .}}<div class="accordian">{{range .}}
+{{define "Examples"}}{{if .}}<div class="panel-group">{{range .}}
-<div class="accordion-group">
+<div class="panel panel-default" id="example-{{.Id}}">
-  <div class="accordion-heading"><a class="accordion-toggle" data-toggle="collapse" id="example-{{.Id}}" href="#ex-{{.Id}}">Example{{with .Example.Name}} ({{.}}){{end}}</a></div>
+  <div class="panel-heading"><a class="accordion-toggle" data-toggle="collapse" href="#ex-{{.Id}}">Example{{with .Example.Name}} ({{.}}){{end}}</a></div>
-  <div id="ex-{{.Id}}" class="accordion-body collapse"><div class="accordion-inner">
+  <div id="ex-{{.Id}}" class="panel-collapse collapse"><div class="panel-body">
     {{with .Example.Doc}}<p>{{.|comment}}{{end}}
     <p>Code:{{if .Example.Play}}<span class="pull-right"><a href="?play={{.Id}}">play</a>&nbsp;</span>{{end}}
     <pre>{{code .Example.Code nil}}</pre>

gddo-server/assets/templates/results.html

@@ -4,6 +4,8 @@
   <div class="well">
     {{template "SearchBox" .q}}
   </div>
+  <p>Search on <a href="http://go-search.org/search?q={{.q}}">Go-Search</a> 
+  or <a href="https://github.com/search?q={{.q}}+language:go">GitHub</a>.
   {{if .pkgs}}
     {{template "Pkgs" .pkgs}}
   {{else}}

gddo-server/assets/third_party/bootstrap/js/bootstrap.js

@@ -314,6 +314,7 @@ if (!jQuery) { throw new Error("Bootstrap requires jQuery") }
   Carousel.DEFAULTS = {
     interval: 5000
   , pause: 'hover'
+  , wrap: true
   }
 
   Carousel.prototype.cycle =  function (e) {
@@ -378,12 +379,15 @@ if (!jQuery) { throw new Error("Bootstrap requires jQuery") }
     var fallback  = type == 'next' ? 'first' : 'last'
     var that      = this
 
+    if (!$next.length) {
+      if (!this.options.wrap) return
+      $next = this.$element.find('.item')[fallback]()
+    }
+
     this.sliding = true
 
     isCycling && this.pause()
 
-    $next = $next.length ? $next : this.$element.find('.item')[fallback]()
-
     var e = $.Event('slide.bs.carousel', { relatedTarget: $next[0], direction: direction })
 
     if ($next.hasClass('active')) return
@@ -535,7 +539,7 @@ if (!jQuery) { throw new Error("Bootstrap requires jQuery") }
     this.$element.trigger(startEvent)
     if (startEvent.isDefaultPrevented()) return
 
-    var actives = this.$parent && this.$parent.find('> .accordion-group > .in')
+    var actives = this.$parent && this.$parent.find('> .panel > .in')
 
     if (actives && actives.length) {
       var hasData = actives.data('bs.collapse')
@@ -656,7 +660,7 @@ if (!jQuery) { throw new Error("Bootstrap requires jQuery") }
     var $parent = parent && $(parent)
 
     if (!data || !data.transitioning) {
-      if ($parent) $parent.find('[data-toggle=collapse][data-parent=' + parent + ']').not($this).addClass('collapsed')
+      if ($parent) $parent.find('[data-toggle=collapse][data-parent="' + parent + '"]').not($this).addClass('collapsed')
       $this[$target.hasClass('in') ? 'addClass' : 'removeClass']('collapsed')
     }
 
@@ -707,7 +711,7 @@ if (!jQuery) { throw new Error("Bootstrap requires jQuery") }
     clearMenus()
 
     if (!isActive) {
-      if ('ontouchstart' in document.documentElement) {
+      if ('ontouchstart' in document.documentElement && !$parent.closest('.navbar-nav').length) {
         // if mobile we we use a backdrop because click events don't delegate
         $('<div class="dropdown-backdrop"/>').insertAfter($(this)).on('click', clearMenus)
       }
@@ -719,9 +723,9 @@ if (!jQuery) { throw new Error("Bootstrap requires jQuery") }
       $parent
         .toggleClass('open')
         .trigger('shown.bs.dropdown')
-    }
 
       $this.focus()
+    }
 
     return false
   }
@@ -847,11 +851,11 @@ if (!jQuery) { throw new Error("Bootstrap requires jQuery") }
 
   var Modal = function (element, options) {
     this.options   = options
-    this.$element  = $(element).on('click.dismiss.modal', '[data-dismiss="modal"]', $.proxy(this.hide, this))
+    this.$element  = $(element)
     this.$backdrop =
     this.isShown   = null
 
-    if (this.options.remote) this.$element.find('.modal-body').load(this.options.remote)
+    if (this.options.remote) this.$element.load(this.options.remote)
   }
 
   Modal.DEFAULTS = {
@@ -860,13 +864,13 @@ if (!jQuery) { throw new Error("Bootstrap requires jQuery") }
     , show: true
   }
 
-  Modal.prototype.toggle = function () {
+  Modal.prototype.toggle = function (_relatedTarget) {
-    return this[!this.isShown ? 'show' : 'hide']()
+    return this[!this.isShown ? 'show' : 'hide'](_relatedTarget)
   }
 
-  Modal.prototype.show = function () {
+  Modal.prototype.show = function (_relatedTarget) {
     var that = this
-    var e    = $.Event('show.bs.modal')
+    var e    = $.Event('show.bs.modal', { relatedTarget: _relatedTarget })
 
     this.$element.trigger(e)
 
@@ -876,6 +880,8 @@ if (!jQuery) { throw new Error("Bootstrap requires jQuery") }
 
     this.escape()
 
+    this.$element.on('click.dismiss.modal', '[data-dismiss="modal"]', $.proxy(this.hide, this))
+
     this.backdrop(function () {
       var transition = $.support.transition && that.$element.hasClass('fade')
 
@@ -895,13 +901,15 @@ if (!jQuery) { throw new Error("Bootstrap requires jQuery") }
 
       that.enforceFocus()
 
+      var e = $.Event('shown.bs.modal', { relatedTarget: _relatedTarget })
+
       transition ?
-        that.$element
+        that.$element.find('.modal-dialog') // wait for modal to slide in
           .one($.support.transition.end, function () {
-            that.$element.focus().trigger('shown.bs.modal')
+            that.$element.focus().trigger(e)
           })
           .emulateTransitionEnd(300) :
-        that.$element.focus().trigger('shown.bs.modal')
+        that.$element.focus().trigger(e)
     })
   }
 
@@ -923,6 +931,7 @@ if (!jQuery) { throw new Error("Bootstrap requires jQuery") }
     this.$element
       .removeClass('in')
       .attr('aria-hidden', true)
+      .off('click.dismiss.modal')
 
     $.support.transition && this.$element.hasClass('fade') ?
       this.$element
@@ -975,7 +984,7 @@ if (!jQuery) { throw new Error("Bootstrap requires jQuery") }
       this.$backdrop = $('<div class="modal-backdrop ' + animate + '" />')
         .appendTo(document.body)
 
-      this.$element.on('click', $.proxy(function (e) {
+      this.$element.on('click.dismiss.modal', $.proxy(function (e) {
         if (e.target !== e.currentTarget) return
         this.options.backdrop == 'static'
           ? this.$element[0].focus.call(this.$element[0])
@@ -1014,15 +1023,15 @@ if (!jQuery) { throw new Error("Bootstrap requires jQuery") }
 
   var old = $.fn.modal
 
-  $.fn.modal = function (option) {
+  $.fn.modal = function (option, _relatedTarget) {
     return this.each(function () {
       var $this   = $(this)
       var data    = $this.data('bs.modal')
       var options = $.extend({}, Modal.DEFAULTS, $this.data(), typeof option == 'object' && option)
 
       if (!data) $this.data('bs.modal', (data = new Modal(this, options)))
-      if (typeof option == 'string') data[option]()
+      if (typeof option == 'string') data[option](_relatedTarget)
-      else if (options.show) data.show()
+      else if (options.show) data.show(_relatedTarget)
     })
   }
 
@@ -1050,23 +1059,21 @@ if (!jQuery) { throw new Error("Bootstrap requires jQuery") }
     e.preventDefault()
 
     $target
-      .modal(option)
+      .modal(option, this)
       .one('hide', function () {
         $this.is(':visible') && $this.focus()
       })
   })
 
-  $(function () {
+  $(document)
-    var $body = $(document.body)
+    .on('show.bs.modal',  '.modal', function () { $(document.body).addClass('modal-open') })
-      .on('shown.bs.modal',  '.modal', function () { $body.addClass('modal-open') })
+    .on('hidden.bs.modal', '.modal', function () { $(document.body).removeClass('modal-open') })
-      .on('hidden.bs.modal', '.modal', function () { $body.removeClass('modal-open') })
-  })
 
 }(window.jQuery);
 
 /* ========================================================================
  * Bootstrap: tooltip.js v3.0.0
- * http://twbs.github.com/bootstrap/javascript.html#affix
+ * http://twbs.github.com/bootstrap/javascript.html#tooltip
  * Inspired by the original jQuery.tipsy by Jason Frame
  * ========================================================================
  * Copyright 2012 Twitter, Inc.
@@ -1157,22 +1164,27 @@ if (!jQuery) { throw new Error("Bootstrap requires jQuery") }
     return options
   }
 
-  Tooltip.prototype.enter = function (obj) {
+  Tooltip.prototype.getDelegateOptions = function () {
-    var defaults = this.getDefaults()
     var options  = {}
+    var defaults = this.getDefaults()
 
     this._options && $.each(this._options, function (key, value) {
       if (defaults[key] != value) options[key] = value
     })
 
+    return options
+  }
+
+  Tooltip.prototype.enter = function (obj) {
     var self = obj instanceof this.constructor ?
-      obj : $(obj.currentTarget)[this.type](options).data('bs.' + this.type)
+      obj : $(obj.currentTarget)[this.type](this.getDelegateOptions()).data('bs.' + this.type)
 
     clearTimeout(self.timeout)
 
+    self.hoverState = 'in'
+
     if (!self.options.delay || !self.options.delay.show) return self.show()
 
-    self.hoverState = 'in'
     self.timeout = setTimeout(function () {
       if (self.hoverState == 'in') self.show()
     }, self.options.delay.show)
@@ -1180,13 +1192,14 @@ if (!jQuery) { throw new Error("Bootstrap requires jQuery") }
 
   Tooltip.prototype.leave = function (obj) {
     var self = obj instanceof this.constructor ?
-      obj : $(obj.currentTarget)[this.type](this._options).data('bs.' + this.type)
+      obj : $(obj.currentTarget)[this.type](this.getDelegateOptions()).data('bs.' + this.type)
 
     clearTimeout(self.timeout)
 
+    self.hoverState = 'out'
+
     if (!self.options.delay || !self.options.delay.hide) return self.hide()
 
-    self.hoverState = 'out'
     self.timeout = setTimeout(function () {
       if (self.hoverState == 'out') self.hide()
     }, self.options.delay.hide)
@@ -1245,7 +1258,7 @@ if (!jQuery) { throw new Error("Bootstrap requires jQuery") }
           .addClass(placement)
       }
 
-      var calculatedOffset = this.getCalcuatedOffset(placement, pos, actualWidth, actualHeight)
+      var calculatedOffset = this.getCalculatedOffset(placement, pos, actualWidth, actualHeight)
 
       this.applyPlacement(calculatedOffset, placement)
       this.$element.trigger('shown.bs.' + this.type)
@@ -1320,7 +1333,9 @@ if (!jQuery) { throw new Error("Bootstrap requires jQuery") }
     var $tip = this.tip()
     var e    = $.Event('hide.bs.' + this.type)
 
-    function complete() { $tip.detach() }
+    function complete() {
+      if (that.hoverState != 'in') $tip.detach()
+    }
 
     this.$element.trigger(e)
 
@@ -1358,7 +1373,7 @@ if (!jQuery) { throw new Error("Bootstrap requires jQuery") }
     }, this.$element.offset())
   }
 
-  Tooltip.prototype.getCalcuatedOffset = function (placement, pos, actualWidth, actualHeight) {
+  Tooltip.prototype.getCalculatedOffset = function (placement, pos, actualWidth, actualHeight) {
     return placement == 'bottom' ? { top: pos.top + pos.height,   left: pos.left + pos.width / 2 - actualWidth / 2  } :
            placement == 'top'    ? { top: pos.top - actualHeight, left: pos.left + pos.width / 2 - actualWidth / 2  } :
            placement == 'left'   ? { top: pos.top + pos.height / 2 - actualHeight / 2, left: pos.left - actualWidth } :
@@ -1405,7 +1420,7 @@ if (!jQuery) { throw new Error("Bootstrap requires jQuery") }
   }
 
   Tooltip.prototype.toggle = function (e) {
-    var self = e ? $(e.currentTarget)[this.type](this._options).data('bs.' + this.type) : this
+    var self = e ? $(e.currentTarget)[this.type](this.getDelegateOptions()).data('bs.' + this.type) : this
     self.tip().hasClass('in') ? self.leave(self) : self.enter(self)
   }
 
@@ -1503,7 +1518,9 @@ if (!jQuery) { throw new Error("Bootstrap requires jQuery") }
 
     $tip.removeClass('fade top bottom left right in')
 
-    $tip.find('.popover-title:empty').hide()
+    // IE8 doesn't accept hiding via the `:empty` pseudo selector, we have to do
+    // this manually by checking the contents.
+    if (!$tip.find('.popover-title').html()) $tip.find('.popover-title').hide()
   }
 
   Popover.prototype.hasContent = function () {

gddo-server/background.go

@@ -65,15 +65,15 @@ func runBackgroundTasks() {
 
 func doCrawl() error {
 	// Look for new package to crawl.
-	importPath, err := db.GetNewCrawl()
+	importPath, hasSubdirs, err := db.PopNewCrawl()
 	if err != nil {
-		log.Printf("db.GetNewCrawl() returned error %v", err)
+		log.Printf("db.PopNewCrawl() returned error %v", err)
 		return nil
 	}
 	if importPath != "" {
-		if pdoc, err := crawlDoc("new", importPath, nil, false, time.Time{}); err != nil || pdoc == nil {
+		if pdoc, err := crawlDoc("new", importPath, nil, hasSubdirs, time.Time{}); pdoc == nil && err == nil {
-			if err := db.SetBadCrawl(importPath); err != nil {
+			if err := db.AddBadCrawl(importPath); err != nil {
-				log.Printf("ERROR db.SetBadCrawl(%q): %v", importPath, err)
+				log.Printf("ERROR db.AddBadCrawl(%q): %v", importPath, err)
 			}
 		}
 		return nil

gddo-server/crawl.go

@@ -34,10 +34,10 @@ func exists(path string) bool {
 }
 
 // crawlDoc fetches the package documentation from the VCS and updates the database.
-func crawlDoc(source string, path string, pdoc *doc.Package, hasSubdirs bool, nextCrawl time.Time) (*doc.Package, error) {
+func crawlDoc(source string, importPath string, pdoc *doc.Package, hasSubdirs bool, nextCrawl time.Time) (*doc.Package, error) {
 	message := []interface{}{source}
 	defer func() {
-		message = append(message, path)
+		message = append(message, importPath)
 		log.Println(message...)
 	}()
 
@@ -56,49 +56,56 @@ func crawlDoc(source string, path string, pdoc *doc.Package, hasSubdirs bool, ne
 
 	start := time.Now()
 	var err error
-	if i := strings.Index(path, "/src/pkg/"); i > 0 && doc.IsGoRepoPath(path[i+len("/src/pkg/"):]) {
+	if i := strings.Index(importPath, "/src/pkg/"); i > 0 && doc.IsGoRepoPath(importPath[i+len("/src/pkg/"):]) {
 		// Go source tree mirror.
 		pdoc = nil
 		err = doc.NotFoundError{Message: "Go source tree mirror."}
-	} else if i := strings.Index(path, "/libgo/go/"); i > 0 && doc.IsGoRepoPath(path[i+len("/libgo/go/"):]) {
+	} else if i := strings.Index(importPath, "/libgo/go/"); i > 0 && doc.IsGoRepoPath(importPath[i+len("/libgo/go/"):]) {
 		// Go Frontend source tree mirror.
 		pdoc = nil
 		err = doc.NotFoundError{Message: "Go Frontend source tree mirror."}
-	} else if m := nestedProjectPat.FindStringIndex(path); m != nil && exists(path[m[0]+1:]) {
+	} else if m := nestedProjectPat.FindStringIndex(importPath); m != nil && exists(importPath[m[0]+1:]) {
 		pdoc = nil
 		err = doc.NotFoundError{Message: "Copy of other project."}
-	} else if blocked, e := db.IsBlocked(path); blocked && e == nil {
+	} else if blocked, e := db.IsBlocked(importPath); blocked && e == nil {
 		pdoc = nil
 		err = doc.NotFoundError{Message: "Blocked."}
 	} else {
 		var pdocNew *doc.Package
-		pdocNew, err = doc.Get(httpClient, path, etag)
+		pdocNew, err = doc.Get(httpClient, importPath, etag)
 		message = append(message, "fetch:", int64(time.Since(start)/time.Millisecond))
-		if err != doc.ErrNotModified {
+		if err == nil && pdocNew.Name == "" && !hasSubdirs {
+			pdoc = nil
+			err = doc.NotFoundError{Message: "No Go files or subdirs"}
+		} else if err != doc.ErrNotModified {
 			pdoc = pdocNew
 		}
 	}
 
 	nextCrawl = start.Add(*maxAge)
-	if strings.HasPrefix(path, "github.com/") || (pdoc != nil && len(pdoc.Errors) > 0) {
+	switch {
+	case strings.HasPrefix(importPath, "github.com/") || (pdoc != nil && len(pdoc.Errors) > 0):
 		nextCrawl = start.Add(*maxAge * 7)
+	case strings.HasPrefix(importPath, "gist.github.com/"):
+		// Don't spend time on gists. It's silly thing to do.
+		nextCrawl = start.Add(*maxAge * 30)
 	}
 
 	switch {
 	case err == nil:
 		message = append(message, "put:", pdoc.Etag)
 		if err := db.Put(pdoc, nextCrawl); err != nil {
-			log.Printf("ERROR db.Put(%q): %v", path, err)
+			log.Printf("ERROR db.Put(%q): %v", importPath, err)
 		}
 	case err == doc.ErrNotModified:
 		message = append(message, "touch")
 		if err := db.SetNextCrawlEtag(pdoc.ProjectRoot, pdoc.Etag, nextCrawl); err != nil {
-			log.Printf("ERROR db.SetNextCrawl(%q): %v", path, err)
+			log.Printf("ERROR db.SetNextCrawl(%q): %v", importPath, err)
 		}
 	case doc.IsNotFound(err):
 		message = append(message, "notfound:", err)
-		if err := db.Delete(path); err != nil {
+		if err := db.Delete(importPath); err != nil {
-			log.Printf("ERROR db.Delete(%q): %v", path, err)
+			log.Printf("ERROR db.Delete(%q): %v", importPath, err)
 		}
 	default:
 		message = append(message, "ERROR:", err)

gddo-server/main.go

@@ -130,11 +130,27 @@ func templateExt(req *web.Request) string {
 }
 
 var (
-	robotPat = regexp.MustCompile(`(:?\+https?://)|(?:\Wbot\W)`)
+	robotPat = regexp.MustCompile(`(:?\+https?://)|(?:\Wbot\W)|(?:^Python-urllib)|(?:^Go )|(?:^Java/)`)
 )
 
 func isRobot(req *web.Request) bool {
-	return *robot || robotPat.MatchString(req.Header.Get(web.HeaderUserAgent))
+	if robotPat.MatchString(req.Header.Get(web.HeaderUserAgent)) {
+		return true
+	}
+	host := req.RemoteAddr
+	if h, _, err := net.SplitHostPort(req.RemoteAddr); err == nil {
+		host = h
+	}
+	n, err := db.IncrementCounter(host, 1)
+	if err != nil {
+		log.Printf("error incrementing counter for %s,  %v\n", host, err)
+		return false
+	}
+	if n > *robot {
+		log.Printf("robot %.2f %s %s", n, host, req.Header.Get(web.HeaderUserAgent))
+		return true
+	}
+	return false
 }
 
 func popularLinkReferral(req *web.Request) bool {
@@ -300,7 +316,12 @@ func servePackage(resp web.Response, req *web.Request) error {
 		if err != nil {
 			return err
 		}
-		return executeTemplate(resp, "importers.html", web.StatusOK, nil, map[string]interface{}{
+		template := "importers.html"
+		if requestType == robotRequest {
+			// Hide back links from robots.
+			template = "importers_robot.html"
+		}
+		return executeTemplate(resp, template, web.StatusOK, nil, map[string]interface{}{
 			"pkgs": pkgs,
 			"pdoc": newTDoc(pdoc),
 		})
@@ -736,7 +757,7 @@ func defaultBase(path string) string {
 
 var (
 	db              *database.Database
-	robot           = flag.Bool("robot", false, "Robot mode")
+	robot           = flag.Float64("robot", 100, "Request counter threshold for robots")
 	assetsDir       = flag.String("assets", filepath.Join(defaultBase("github.com/garyburd/gddo/gddo-server"), "assets"), "Base directory for templates and static files.")
 	gzAssetsDir     = flag.String("gzassets", "", "Base directory for compressed static files.")
 	presentDir      = flag.String("present", defaultBase("code.google.com/p/go.talks/present"), "Base directory for templates and static files.")
@@ -827,6 +848,7 @@ func main() {
 		{"dir.html", "common.html", "layout.html"},
 		{"home.html", "common.html", "layout.html"},
 		{"importers.html", "common.html", "layout.html"},
+		{"importers_robot.html", "common.html", "layout.html"},
 		{"imports.html", "common.html", "layout.html"},
 		{"file.html", "common.html", "layout.html"},
 		{"index.html", "common.html", "layout.html"},
@@ -946,7 +968,10 @@ func main() {
 		return
 	}
 	defer listener.Close()
-	s := &server.Server{Listener: listener, Handler: h} // add logger
+	s := &server.Server{
+		Listener: listener,
+		Handler:  web.ProxyHeaderHandler("X-Real-Ip", "X-Scheme", h),
+	}
 	err = s.Serve()
 	if err != nil {
 		log.Fatal("Server", err)

gddo-server/template.go

@@ -157,7 +157,8 @@ func (pdoc *tdoc) Breadcrumbs(templateName string) htemp.HTML {
 		if i != 0 {
 			buf.WriteString(`<span class="text-muted">/</span>`)
 		}
-		link := j < len(pdoc.ImportPath) || (templateName != "cmd.html" && templateName != "pkg.html")
+		link := j < len(pdoc.ImportPath) ||
+			(templateName != "dir.html" && templateName != "cmd.html" && templateName != "pkg.html")
 		if link {
 			buf.WriteString(`<a href="`)
 			buf.WriteString(formatPathFrag(pdoc.ImportPath[:j], ""))