internal/source: trim .git suffix from repo URLs

Remove the ".git" from the repo URL of source templates
when we know it will result in invalid URLs.

The ".git" is understood by the Go command, but not
by some code hosting sites. To be safe, only remove
it on code hosting sites we've tested. For now,
that's GitHub and GitLab.

Remove the suffix in three places:

1. When we construct a SourceInfo from the repo URL and meta
   tags. This will fix newly processed modules.

2. When we unmarshal a SourceInfo from the DB. This will fix it during
   rendering for already-processed modules. It will save us from having
   to reprocess or backfill.

3. In the function we use in tests.

Fixes golang/go#44032

Change-Id: Ic6d48d0422dc12ce8dbc4586f8d6c586f463531e
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/288612
Trust: Jonathan Amsterdam <jba@google.com>
Run-TryBot: Jonathan Amsterdam <jba@google.com>
TryBot-Result: kokoro <noreply+kokoro@google.com>
Reviewed-by: Jamal Carvalho <jamal@golang.org>

internal/frontend/readme_test.go

@@ -199,6 +199,18 @@ func TestReadme(t *testing.T) {
 			wantHTML:    `<p><img src="http://github.com/golang/go/raw/master/doc/logo.png" alt="Go logo"/></p>`,
 			wantOutline: nil,
 		},
+		{
+			name: "relative image markdown is made absolute for GitHub, .git removed from repo URL",
+			unit: &internal.Unit{
+				UnitMeta: internal.UnitMeta{SourceInfo: source.NewGitHubInfo("https://github.com/robpike/ivy.git", "", "v0.1.0")},
+			},
+			readme: &internal.Readme{
+				Filepath: "README.md",
+				Contents: "![ivy](ivy.jpg)",
+			},
+			wantHTML:    `<p><img src="https://github.com/robpike/ivy/raw/v0.1.0/ivy.jpg" alt="ivy"/></p>`,
+			wantOutline: nil,
+		},
 		{
 			name: "relative image markdown is left alone for unknown origins",
 			unit: &internal.Unit{},

internal/source/source.go

@@ -192,7 +192,7 @@ func (i *Info) UnmarshalJSON(data []byte) (err error) {
 	if err := json.Unmarshal(data, &ji); err != nil {
 		return err
 	}
-	i.repoURL = ji.RepoURL
+	i.repoURL = trimVCSSuffix(ji.RepoURL)
 	i.moduleDir = ji.ModuleDir
 	i.commit = ji.Commit
 	if ji.Kind != "" {
@@ -275,7 +275,7 @@ func ModuleInfo(ctx context.Context, client *Client, modulePath, version string)
 			commit = transformCommit(commit, isHash)
 		}
 		info = &Info{
-			repoURL:   "https://" + repo,
+			repoURL:   trimVCSSuffix("https://" + repo),
 			moduleDir: relativeModulePath,
 			commit:    commit,
 			templates: templates,
@@ -755,6 +755,26 @@ func commitFromVersion(vers, relativeModulePath string) (commit string, isHash b
 	}
 }
 
+// trimVCSSuffix removes a VCS suffix from a repo URL in selected cases.
+//
+// The Go command allows a VCS suffix on a repo, like github.com/foo/bar.git. But
+// some code hosting sites don't support all paths constructed from such URLs.
+// For example, GitHub will redirect github.com/foo/bar.git to github.com/foo/bar,
+// but will 404 on github.com/goo/bar.git/tree/master and any other URL with a
+// non-empty path.
+//
+// To be conservative, we remove the suffix only in cases where we know it's
+// wrong.
+func trimVCSSuffix(repoURL string) string {
+	if !strings.HasSuffix(repoURL, ".git") {
+		return repoURL
+	}
+	if strings.HasPrefix(repoURL, "https://github.com/") || strings.HasPrefix(repoURL, "https://gitlab.com/") {
+		return strings.TrimSuffix(repoURL, ".git")
+	}
+	return repoURL
+}
+
 // The following code copied from cmd/go/internal/get:
 
 // expand rewrites s to replace {k} with match[k] for each key k in match.
@@ -773,7 +793,7 @@ func expand(s string, match map[string]string) string {
 // It is for testing only.
 func NewGitHubInfo(repoURL, moduleDir, commit string) *Info {
 	return &Info{
-		repoURL:   repoURL,
+		repoURL:   trimVCSSuffix(repoURL),
 		moduleDir: moduleDir,
 		commit:    commit,
 		templates: githubURLTemplates,

internal/source/source_test.go

@@ -93,6 +93,16 @@ func TestModuleInfo(t *testing.T) {
 			"https://github.com/hashicorp/consul/blob/sdk/v0.2.0/sdk/freeport/freeport.go#L1",
 			"https://github.com/hashicorp/consul/raw/sdk/v0.2.0/sdk/freeport/freeport.go",
 		},
+		{
+			"github module with VCS suffix",
+			"github.com/pkg/errors.git", "v0.8.1", "errors.go",
+
+			"https://github.com/pkg/errors",
+			"https://github.com/pkg/errors/tree/v0.8.1",
+			"https://github.com/pkg/errors/blob/v0.8.1/errors.go",
+			"https://github.com/pkg/errors/blob/v0.8.1/errors.go#L1",
+			"https://github.com/pkg/errors/raw/v0.8.1/errors.go",
+		},
 		{
 			"bitbucket",
 			"bitbucket.org/plazzaro/kami", "v1.2.1", "defaults.go",