go
/
pkgsite
зеркало из https://github.com/golang/pkgsite.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

discovery: add handling for non-redistributable packages 

This CL implements admittedly complicated logic for handling license
redistributability. It does this in three main steps, which could
conceivably be split into separate CLs if desirable.

1. Add an internal.Package.IsRedistributable method, which reports a
package as redistributable if and only if there is at least one license
permitting redistribution at the module root, and if every directory
containing a detected license file contains at least one license file
permitting redistribution. It is very possible that this logic is will
change in the future.

2. Add behavior in the postgres handler to prune package and version
data that should not be stored for non-redistributable content. As a
precaution it was decided to do this in the data layer rather than at
fetch time.

3. Add handling in the frontend to make it clear when content is not
redistributable.

Because we now have a scenario where we can't show license details, but
still want to surface the license disclaimer, move it to a separate
page.

Also:
 + modify license extraction to store the license path relative to the
   module@version subdirectory of the module zip.
 + fix a bug where VersionType was not set on in postgres.GetPackage.

Fixes b/127335320
Updates b/124309095

Change-Id: I36194b78651ccff91b0e06d0d3d4fc639a884565
Reviewed-on: https://team-review.git.corp.google.com/c/golang/discovery/+/455069
Reviewed-by: Julie Qiu <julieqiu@google.com>
This commit is contained in:
Rob Findley 2019-04-26 09:58:34 -04:00 коммит произвёл Julie Qiu
Родитель 5b65f2c87d
Коммит 089acc7de6
36 изменённых файлов: 576 добавлений и 147 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

1
cmd/frontend/main.go
Просмотреть файл

@ -47,6 +47,7 @@ func main() {
http.ServeFile(w, r, fmt.Sprintf("%s/img/favicon.ico", http.Dir(*staticPath)))
})
mux.HandleFunc("/search/", controller.HandleSearch)
mux.HandleFunc("/license-policy/", controller.HandleStaticPage("license_policy.tmpl"))
mux.HandleFunc("/", controller.HandleDetails)
mw := middleware.Timeout(1 * time.Minute)

30
content/static/html/pages/details.tmpl
Просмотреть файл

@ -18,21 +18,25 @@
{{ if not $header.IsCommand }}
<p>import "{{ $header.Path }}"</p>
{{ end }}
<p>{{ $header.Synopsis }}</p>
<p>
{{ if $header.IsRedistributable -}}
{{ $header.Synopsis }}
{{- else -}}
<em>Synopsis hidden due to license restrictions.</em>
{{ end }}
</p>
<p>
<b>Version: </b><span>{{ $header.Version }}</span>
<span>|</span>
<b>Published: </b><span>{{ $header.CommitTime }}</span>
<span>|</span>
<b>License(s): </b>
{{ if $header.Licenses }}
{{ range $i, $e := $header.Licenses -}}{{ if $i }}, {{ end }}
<a href="/{{$header.Path}}?v={{$header.Version}}&tab=licenses#{{.Anchor}}">{{ $e.Type }}</a>
{{- end }}
{{ else }}
<span>No license files detected</span>
{{ end }}
<a href="/{{$header.Path}}?v={{$header.Version}}&tab=licenses#Disclaimer" class="Disclaimer-link">(This is not legal advice)</a>
{{ range $i, $e := $header.Licenses -}}{{ if $i }}, {{ end }}
<a href="/{{$header.Path}}?v={{$header.Version}}&tab=licenses#{{.Anchor}}">{{ $e.Type }}</a>
{{- else -}}
<span>No license files detected</span>
{{- end }}
<a href="/license-policy/" class="Disclaimer-link">(This is not legal advice)</a>
</p>
</header>
@ -60,8 +64,12 @@
</li>
</ul>
<div class="Content">
{{ template "content" .Details }}
</div>
{{ if .CanShowDetails -}}
{{ template "content" .Details }}
{{- else }}
<h2>This data is not redistributable due to license restrictions.</h2>
{{ end }}
</div>
{{ template "footer" }}
</div>

27
content/static/html/pages/license_policy.tmpl Normal file
Просмотреть файл

@ -0,0 +1,27 @@
<!--
Copyright 2019 The Go Authors. All rights reserved.
Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file.
-->
{{ define "ROOT" }}
<!DOCTYPE html>
<title>Licenses — Go Discovery</title>
{{ template "header" . }}
{{ template "nav" . }}
<div class="Container">
<h1>License Disclaimer</h1>
<p class="License-disclaimer">
The go.dev website displays license information in order to help users
evaluate packages for their intended use. Licenses are detected based on
the name and contents of the linked license files to the best of our
ability. We hope this information is helpful, but this is not legal advice
and we do not make any guarantees regarding the accuracy of our license
detection.
</p>
</div>
{{ end }}

11
content/static/html/pages/licenses.tmpl
Просмотреть файл

@ -5,17 +5,6 @@
-->
{{ define "content" }}
<section id="Disclaimer">
<h3><a href="#Disclaimer">Disclaimer</a></h3>
<p class="License-disclaimer">
The go.dev website displays license information in order to help users
evaluate packages for their intended use. Licenses are detected based on
the name and contents of the linked license files to the best of our
ability. We hope this information is helpful, but this is not legal advice
and we do not make any guarantees regarding the accuracy of our license
detection.
</p>
</section>
{{ range .Licenses }}
<section class="License" id="{{ .Anchor }}">
<h3><a href="#{{ .Anchor }}">{{ .Type }} <small>({{ .FilePath }})</small></a></h3>

5
internal/discovery.go
Просмотреть файл

@ -48,6 +48,11 @@ type Package struct {
Imports []*Import
}
// IsRedistributable reports whether the package may be redistributed.
func (p *Package) IsRedistributable() bool {
return licensesAreRedistributable(p.Licenses)
}
// VersionedPackage is a Package along with its corresponding version
// information.
type VersionedPackage struct {

23
internal/fetch/fetch.go
Просмотреть файл

@ -128,7 +128,7 @@ func FetchAndInsertVersion(modulePath, version string, proxyClient *proxy.Client
return fmt.Errorf("extractReadmeFromZip(%q, %q, zipReader): %v", modulePath, version, err)
}
licenses, err := detectLicenses(zipReader)
licenses, err := detectLicenses(moduleVersionDir(modulePath, version), zipReader)
if err != nil {
log.Printf("Error detecting licenses for %v@%v: %v", modulePath, version, err)
}
@ -151,6 +151,7 @@ func FetchAndInsertVersion(modulePath, version string, proxyClient *proxy.Client
},
Packages: packages,
}
if err = db.InsertVersion(ctx, v, licenses); err != nil {
return fmt.Errorf("db.InsertVersion for %q %q: %v", modulePath, version, err)
}
@ -160,11 +161,10 @@ func FetchAndInsertVersion(modulePath, version string, proxyClient *proxy.Client
return nil
}
// trimeModuleVersionPrefix trims the prefix <modulePath>@<version>/ from the
// filepath, which is the expected base directory for the contents of a module
// zip from the proxy.
func trimModuleVersionPrefix(modulePath, version, filePath string) string {
return strings.TrimPrefix(filePath, fmt.Sprintf("%s@%s/", modulePath, version))
// moduleVersionDir formats the content subdirectory for the given
// modulePath and version.
func moduleVersionDir(modulePath, version string) string {
return fmt.Sprintf("%s@%s", modulePath, version)
}
// extractReadmeFromZip returns the file path and contents of the first file
@ -177,7 +177,7 @@ func extractReadmeFromZip(modulePath, version string, r *zip.Reader) (string, []
if err != nil {
return "", nil, fmt.Errorf("readZipFile(%q): %v", zipFile.Name, err)
}
return trimModuleVersionPrefix(modulePath, version, zipFile.Name), c, nil
return strings.TrimPrefix(zipFile.Name, moduleVersionDir(modulePath, version)+"/"), c, nil
}
}
return "", nil, errReadmeNotFound
@ -214,7 +214,7 @@ func extractPackagesFromZip(modulePath, version string, r *zip.Reader, licenses
return nil, fmt.Errorf("loadPackages(%q, %q, %q, zipReader): %v", workDir, modulePath, version, err)
}
packages, err := transformPackages(workDir, modulePath, pkgs, licenses)
packages, err := transformPackages(workDir, modulePath, version, pkgs, licenses)
if err != nil {
return nil, fmt.Errorf("transformPackages(%q, %q, pkgs, licenses): %v", workDir, modulePath, err)
}
@ -274,7 +274,7 @@ func writeGoModFile(filename, modulePath string) error {
func loadPackages(workDir, modulePath, version string) ([]*packages.Package, error) {
config := &packages.Config{
Mode: packages.LoadSyntax,
Dir: fmt.Sprintf("%s/%s@%s", workDir, modulePath, version),
Dir: filepath.Join(workDir, moduleVersionDir(modulePath, version)),
}
pattern := fmt.Sprintf("%s/...", modulePath)
pkgs, err := packages.Load(config, pattern)
@ -333,8 +333,9 @@ func (m licenseMatcher) matchLicenses(p *packages.Package) []*internal.LicenseIn
// transformPackages maps a slice of *packages.Package to our internal
// representation (*internal.Package). To do so, it maps package data
// and matches packages with their applicable licenses.
func transformPackages(workDir, modulePath string, pkgs []*packages.Package, licenses []*internal.License) ([]*internal.Package, error) {
matcher := newLicenseMatcher(workDir, licenses)
func transformPackages(workDir, modulePath, version string, pkgs []*packages.Package, licenses []*internal.License) ([]*internal.Package, error) {
licenseDir := filepath.Join(workDir, moduleVersionDir(modulePath, version))
matcher := newLicenseMatcher(licenseDir, licenses)
packages := []*internal.Package{}
if len(pkgs) > maxPackagesPerModule {

116
internal/fetch/fetch_test.go
Просмотреть файл

@ -35,40 +35,95 @@ func TestFetchAndInsertVersion(t *testing.T) {
defer cancel()
testCases := []struct {
modulePath string
version string
versionData *internal.VersionInfo
pkg string
pkgData *internal.Package
modulePath string
version string
pkg string
want *internal.VersionedPackage
}{
{
modulePath: "my.mod/module",
version: "v1.0.0",
versionData: &internal.VersionInfo{
SeriesPath: "my.mod/module",
ModulePath: "my.mod/module",
Version: "v1.0.0",
CommitTime: time.Date(2019, 1, 30, 0, 0, 0, 0, time.UTC),
ReadmeFilePath: "README.md",
ReadmeContents: []byte("README FILE FOR TESTING."),
VersionType: "release",
pkg: "my.mod/module/bar",
want: &internal.VersionedPackage{
VersionInfo: internal.VersionInfo{
SeriesPath: "my.mod/module",
ModulePath: "my.mod/module",
Version: "v1.0.0",
CommitTime: time.Date(2019, 1, 30, 0, 0, 0, 0, time.UTC),
ReadmeFilePath: "README.md",
ReadmeContents: []byte("README FILE FOR TESTING."),
VersionType: "release",
},
Package: internal.Package{
Path: "my.mod/module/bar",
Name: "bar",
Synopsis: "package bar",
Suffix: "bar",
Licenses: []*internal.LicenseInfo{
{Type: "BSD-3-Clause", FilePath: "LICENSE"},
{Type: "MIT", FilePath: "bar/LICENSE"},
},
},
},
pkg: "my.mod/module/bar",
pkgData: &internal.Package{
Path: "my.mod/module/bar",
Name: "bar",
Synopsis: "package bar",
Suffix: "bar",
Licenses: []*internal.LicenseInfo{
{Type: "BSD-3-Clause", FilePath: "my.mod/module@v1.0.0/LICENSE"},
{Type: "MIT", FilePath: "my.mod/module@v1.0.0/bar/LICENSE"},
}, {
modulePath: "nonredistributable.mod/module",
version: "v1.0.0",
pkg: "nonredistributable.mod/module/bar/baz",
want: &internal.VersionedPackage{
VersionInfo: internal.VersionInfo{
SeriesPath: "nonredistributable.mod/module",
ModulePath: "nonredistributable.mod/module",
Version: "v1.0.0",
CommitTime: time.Date(2019, 1, 30, 0, 0, 0, 0, time.UTC),
ReadmeFilePath: "README.md",
ReadmeContents: []byte("README FILE FOR TESTING."),
VersionType: "release",
},
Package: internal.Package{
Path: "nonredistributable.mod/module/bar/baz",
Name: "baz",
Synopsis: "package baz",
Suffix: "bar/baz",
Licenses: []*internal.LicenseInfo{
{Type: "BSD-0-Clause", FilePath: "bar/baz/LICENSE"},
{Type: "BSD-0-Clause", FilePath: "LICENSE.txt"},
{Type: "BSD-3-Clause", FilePath: "LICENSE"},
{Type: "MIT", FilePath: "bar/baz/COPYING"},
{Type: "MIT", FilePath: "bar/LICENSE"},
},
},
},
}, {
modulePath: "nonredistributable.mod/module",
version: "v1.0.0",
pkg: "nonredistributable.mod/module/foo",
want: &internal.VersionedPackage{
VersionInfo: internal.VersionInfo{
SeriesPath: "nonredistributable.mod/module",
ModulePath: "nonredistributable.mod/module",
Version: "v1.0.0",
CommitTime: time.Date(2019, 1, 30, 0, 0, 0, 0, time.UTC),
ReadmeFilePath: "README.md",
ReadmeContents: []byte("README FILE FOR TESTING."),
VersionType: "release",
},
Package: internal.Package{
Path: "nonredistributable.mod/module/foo",
Name: "foo",
Synopsis: "",
Suffix: "foo",
Licenses: []*internal.LicenseInfo{
{Type: "BSD-0-Clause", FilePath: "foo/LICENSE.md"},
{Type: "BSD-0-Clause", FilePath: "LICENSE.txt"},
{Type: "BSD-3-Clause", FilePath: "LICENSE"},
},
},
},
},
}
for _, test := range testCases {
t.Run(test.modulePath, func(t *testing.T) {
t.Run(test.pkg, func(t *testing.T) {
defer postgres.ResetTestDB(testDB, t)
teardownProxy, client := proxy.SetupTestProxy(ctx, t)
@ -78,21 +133,12 @@ func TestFetchAndInsertVersion(t *testing.T) {
t.Fatalf("FetchAndInsertVersion(%q, %q, %v, %v): %v", test.modulePath, test.version, client, testDB, err)
}
dbVersion, err := testDB.GetVersion(ctx, test.modulePath, test.version)
gotVersion, err := testDB.GetVersion(ctx, test.modulePath, test.version)
if err != nil {
t.Fatalf("testDB.GetVersion(ctx, %q, %q): %v", test.modulePath, test.version, err)
}
// create a clone of dbVersion, as we want to use it for package testing later.
got := *dbVersion
// got.CommitTime has a timezone location of +0000, while
// test.versionData.CommitTime has a timezone location of UTC.
// These are equal according to time.Equal, but fail for
// reflect.DeepEqual. Convert the DB time to UTC.
got.CommitTime = got.CommitTime.UTC()
if diff := cmp.Diff(*test.versionData, got); diff != "" {
if diff := cmp.Diff(test.want.VersionInfo, *gotVersion); diff != "" {
t.Errorf("testDB.GetVersion(ctx, %q, %q) mismatch (-want +got):\n%s", test.modulePath, test.version, diff)
}
@ -104,7 +150,7 @@ func TestFetchAndInsertVersion(t *testing.T) {
sort.Slice(gotPkg.Licenses, func(i, j int) bool {
return gotPkg.Licenses[i].Type < gotPkg.Licenses[j].Type
})
if diff := cmp.Diff(test.pkgData, &gotPkg.Package); diff != "" {
if diff := cmp.Diff(test.want, gotPkg); diff != "" {
t.Errorf("testDB.GetPackage(ctx, %q, %q) mismatch (-want +got):\n%s", test.pkg, test.version, diff)
}
})

33
internal/fetch/license.go
Просмотреть файл

@ -20,23 +20,17 @@ const (
licenseCoverageThreshold = 90
)
var (
// licenseFileNames is the list of file names that could contain a license.
licenseFileNames = map[string]bool{
"LICENSE": true,
"LICENSE.md": true,
"COPYING": true,
"COPYING.md": true,
}
)
// detectLicense searches for possible license files in the contents directory
// of the provided zip path, runs them against a license classifier, and provides all
// licenses with a confidence score that meet the licenseClassifyThreshold.
func detectLicenses(r *zip.Reader) ([]*internal.License, error) {
// detectLicense searches for possible license files in a subdirectory within
// the provided zip path, runs them against a license classifier, and provides
// all licenses with a confidence score that meet the licenseClassifyThreshold.
//
// It returns an error if the given file path is invalid, if the uncompressed
// size of the license file is too large, if a license is discovered outside of
// the expected path, or if an error occurs during extraction.
func detectLicenses(subdir string, r *zip.Reader) ([]*internal.License, error) {
var licenses []*internal.License
for _, f := range r.File {
if !licenseFileNames[filepath.Base(f.Name)] || strings.Contains(f.Name, "/vendor/") {
if !internal.LicenseFileNames[filepath.Base(f.Name)] || strings.Contains(f.Name, "/vendor/") {
// Only consider licenses with an acceptable file name, and not in the
// vendor directory.
continue
@ -44,6 +38,13 @@ func detectLicenses(r *zip.Reader) ([]*internal.License, error) {
if err := module.CheckFilePath(f.Name); err != nil {
return nil, fmt.Errorf("module.CheckFilePath(%q): %v", f.Name, err)
}
prefix := ""
if subdir != "" {
prefix = subdir + "/"
}
if !strings.HasPrefix(f.Name, prefix) {
return nil, fmt.Errorf("potential license file %q found outside of the expected path %s", f.Name, subdir)
}
if f.UncompressedSize64 > 1e7 {
return nil, fmt.Errorf("potential license file %q exceeds maximum uncompressed size %d", f.Name, int(1e7))
}
@ -63,7 +64,7 @@ func detectLicenses(r *zip.Reader) ([]*internal.License, error) {
license := &internal.License{
LicenseInfo: internal.LicenseInfo{
Type: m.Name,
FilePath: f.Name,
FilePath: strings.TrimPrefix(f.Name, prefix),
},
Contents: bytes,
}

14
internal/fetch/license_test.go
Просмотреть файл

@ -16,8 +16,8 @@ func TestDetectLicenses(t *testing.T) {
return []*internal.LicenseInfo{{Type: licType, FilePath: licFile}}
}
testCases := []struct {
name, zipName string
want []*internal.LicenseInfo
name, zipName, subdir string
want []*internal.LicenseInfo
}{
{
name: "valid_license",
@ -27,8 +27,12 @@ func TestDetectLicenses(t *testing.T) {
name: "valid_license_md_format",
zipName: "licensemd",
want: makeLicenses("MIT", "rsc.io/quote@v1.4.1/LICENSE.md"),
},
{
}, {
name: "valid_license_trim_prefix",
subdir: "rsc.io/quote@v1.4.1",
zipName: "licensemd",
want: makeLicenses("MIT", "LICENSE.md"),
}, {
name: "valid_license_copying",
zipName: "copying",
want: makeLicenses("Apache-2.0", "golang.org/x/text@v0.0.3/COPYING"),
@ -79,7 +83,7 @@ func TestDetectLicenses(t *testing.T) {
defer rc.Close()
z := &rc.Reader
got, err := detectLicenses(z)
got, err := detectLicenses(test.subdir, z)
if err != nil {
t.Errorf("detectLicenses(z): %v", err)
}

9
internal/frontend/controller.go
Просмотреть файл

@ -34,6 +34,14 @@ func New(db *postgres.DB, templateDir string) (*Controller, error) {
}, nil
}
// HandleStaticPage handles requests to a template that contains no dynamic
// content.
func (c *Controller) HandleStaticPage(templateName string) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
c.renderPage(w, templateName, nil)
}
}
// renderPage is used to execute all templates for a *Controller. It expects
// the file for templateName to be defined as "ROOT".
func (c *Controller) renderPage(w http.ResponseWriter, templateName string, page interface{}) {
@ -59,6 +67,7 @@ func parsePageTemplates(base string) (map[string]*template.Template, error) {
{"index.tmpl"},
{"package404.tmpl"},
{"search.tmpl"},
{"license_policy.tmpl"},
{"doc.tmpl", "details.tmpl"},
{"importedby.tmpl", "details.tmpl"},
{"imports.tmpl", "details.tmpl"},

127
internal/frontend/details.go
Просмотреть файл

@ -27,8 +27,9 @@ import (
// DetailsPage contains data for the doc template.
type DetailsPage struct {
Details interface{}
PackageHeader *Package
CanShowDetails bool
Details interface{}
PackageHeader *Package
}
// OverviewDetails contains all of the data that the overview template
@ -112,16 +113,17 @@ type MinorVersionGroup struct {
// Package contains information for an individual package.
type Package struct {
Name string
Version string
Path string
ModulePath string
Synopsis string
CommitTime string
Title string
Suffix string
Licenses []LicenseInfo
IsCommand bool
Name string
Version string
Path string
ModulePath string
Synopsis string
CommitTime string
Title string
Suffix string
Licenses []LicenseInfo
IsCommand bool
IsRedistributable bool
}
// transformLicenseInfos transforms an internal.LicenseInfo into a LicenseInfo,
@ -150,15 +152,16 @@ func createPackageHeader(pkg *internal.VersionedPackage) (*Package, error) {
}
name := packageName(pkg.Name, pkg.Path)
return &Package{
Name: name,
IsCommand: isCmd,
Title: packageTitle(name, isCmd),
Version: pkg.VersionInfo.Version,
Path: pkg.Path,
Synopsis: pkg.Package.Synopsis,
Suffix: pkg.Package.Suffix,
Licenses: transformLicenseInfos(pkg.Licenses),
CommitTime: elapsedTime(pkg.VersionInfo.CommitTime),
Name: name,
IsCommand: isCmd,
Title: packageTitle(name, isCmd),
Version: pkg.VersionInfo.Version,
Path: pkg.Path,
Synopsis: pkg.Package.Synopsis,
Suffix: pkg.Package.Suffix,
Licenses: transformLicenseInfos(pkg.Licenses),
CommitTime: elapsedTime(pkg.VersionInfo.CommitTime),
IsRedistributable: pkg.IsRedistributable(),
}, nil
}
@ -429,6 +432,42 @@ func readmeHTML(readmeFilePath string, readmeContents []byte) template.HTML {
return template.HTML(string(p.SanitizeBytes(unsafe)))
}
// tabSettings defines rendering options associated to each tab. Any tab value
// not present in this map will be handled as a request to 'overview'.
var tabSettings = map[string]struct {
alwaysShowDetails bool
}{
"doc": {},
"importedby": {alwaysShowDetails: true},
"imports": {alwaysShowDetails: true},
"licenses": {},
"module": {},
"overview": {},
"versions": {alwaysShowDetails: true},
}
// fetchDetails returns tab details by delegating to the correct detail
// handler.
func fetchDetails(ctx context.Context, tab string, db *postgres.DB, pkg *internal.VersionedPackage) (interface{}, error) {
switch tab {
case "doc":
return fetchDocumentationDetails(ctx, db, pkg)
case "versions":
return fetchVersionsDetails(ctx, db, &pkg.Package)
case "module":
return fetchModuleDetails(ctx, db, pkg)
case "imports":
return fetchImportsDetails(ctx, db, pkg)
case "importedby":
return fetchImportedByDetails(ctx, db, &pkg.Package)
case "licenses":
return fetchLicensesDetails(ctx, db, pkg)
case "overview":
return fetchOverviewDetails(ctx, db, pkg)
}
return nil, fmt.Errorf("BUG: unable to fetch details: unknown tab %q", tab)
}
// HandleDetails applies database data to the appropriate template. Handles all
// endpoints that match "/" or "/<import-path>[@<version>?tab=<tab>]"
func (c *Controller) HandleDetails(w http.ResponseWriter, r *http.Request) {
@ -452,10 +491,9 @@ func (c *Controller) HandleDetails(w http.ResponseWriter, r *http.Request) {
}
var (
details interface{}
pkg *internal.VersionedPackage
err error
ctx = r.Context()
pkg *internal.VersionedPackage
err error
ctx = r.Context()
)
if version == "" {
@ -482,35 +520,28 @@ func (c *Controller) HandleDetails(w http.ResponseWriter, r *http.Request) {
}
tab := r.FormValue("tab")
switch tab {
case "doc":
details, err = fetchDocumentationDetails(ctx, c.db, pkg)
case "versions":
details, err = fetchVersionsDetails(ctx, c.db, &pkg.Package)
case "module":
details, err = fetchModuleDetails(ctx, c.db, pkg)
case "imports":
details, err = fetchImportsDetails(ctx, c.db, pkg)
case "importedby":
details, err = fetchImportedByDetails(ctx, c.db, &pkg.Package)
case "licenses":
details, err = fetchLicensesDetails(ctx, c.db, pkg)
case "overview":
fallthrough
default:
settings, ok := tabSettings[tab]
if !ok {
tab = "overview"
details, err = fetchOverviewDetails(ctx, c.db, pkg)
settings = tabSettings["overview"]
}
canShowDetails := pkg.IsRedistributable() || settings.alwaysShowDetails
if err != nil {
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
log.Printf("error fetching page for %q: %v", tab, err)
return
var details interface{}
if canShowDetails {
var err error
details, err = fetchDetails(ctx, tab, c.db, pkg)
if err != nil {
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
log.Printf("error fetching page for %q: %v", tab, err)
return
}
}
page := &DetailsPage{
PackageHeader: pkgHeader,
Details: details,
PackageHeader: pkgHeader,
Details: details,
CanShowDetails: canShowDetails,
}
c.renderPage(w, tab+".tmpl", page)

1
internal/frontend/search_test.go
Просмотреть файл

@ -53,6 +53,7 @@ func TestFetchSearchPage(t *testing.T) {
Name: "bar",
Path: "/path/to/bar",
Synopsis: "bar is used by foo.",
Licenses: sampleLicenseInfos,
},
},
}

75
internal/license.go Normal file
Просмотреть файл

@ -0,0 +1,75 @@
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package internal
import "path"
// RedistributableLicenses defines the set of licenses that are permissive of
// redistribution.
var RedistributableLicenses = map[string]bool{
"Apache-2.0": true,
"Artistic-2.0": true,
"BSD-2-Clause-FreeBSD": true,
"BSD-3-Clause": true,
"BSL-1.0": true,
"CC-BY-4.0": true,
"CC0-1.0": true,
"GPL2": true,
"GPL3": true,
"ISC": true,
"JSON": true,
"LGPL-2.1": true,
"LGPL-3.0": true,
"MIT": true,
"Unlicense": true,
"Zlib": true,
}
// LicenseFileNames defines the set of filenames to be considered for license
// extraction.
var LicenseFileNames = map[string]bool{
"LICENSE": true,
"LICENSE.md": true,
"LICENSE.txt": true,
"COPYING": true,
"COPYING.md": true,
}
// licensesAreRedistributable determines whether content subject to the given
// licenses should be considered redistributable. The current algorithm for
// this is to ensure that (1) There is at least one license permitting
// redistribution in the root directory, and (2) every directory containing an
// applicable license contains at least one license that is redistributable.
func licensesAreRedistributable(licenses []*LicenseInfo) bool {
byDir := make(map[string][]string)
for _, l := range licenses {
dir := path.Dir(l.FilePath)
byDir[dir] = append(byDir[dir], l.Type)
}
anyRedistributable := func(lics []string) bool {
for _, l := range lics {
if RedistributableLicenses[l] {
return true
}
}
return false
}
// There must be a license at the module level, otherwise it's can't be
// redistributable. We'll check if any top-level license is redistributable
// below.
if len(byDir["."]) == 0 {
return false
}
for _, lics := range byDir {
if !anyRedistributable(lics) {
return false
}
}
return true
}

70
internal/license_test.go Normal file
Просмотреть файл

@ -0,0 +1,70 @@
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package internal
import "testing"
func TestLicensesAreRedistributable(t *testing.T) {
tests := []struct {
label string
licenses []*LicenseInfo
want bool
}{
{
label: "redistributable license",
licenses: []*LicenseInfo{
{Type: "MIT", FilePath: "LICENSE"},
},
want: true,
}, {
label: "no redistributable license at root",
licenses: []*LicenseInfo{
{Type: "MIT", FilePath: "bar/LICENSE"},
},
want: false,
}, {
label: "no license",
want: false,
}, {
label: "non-redistributable license",
licenses: []*LicenseInfo{
{Type: "AGPL-3.0", FilePath: "LICENSE"},
},
want: false,
}, {
label: "multiple redistributable",
licenses: []*LicenseInfo{
{Type: "BSD-3-Clause", FilePath: "LICENSE"},
{Type: "MIT", FilePath: "bar/LICENSE"},
},
want: true,
}, {
label: "not all redistributable",
licenses: []*LicenseInfo{
{Type: "BSD-3-Clause", FilePath: "LICENSE"},
{Type: "AGPL-3.0", FilePath: "foo/LICENSE"},
{Type: "MIT", FilePath: "foo/bar/LICENSE"},
},
want: false,
}, {
label: "at least one redistributable per directory",
licenses: []*LicenseInfo{
{Type: "BSD-3-Clause", FilePath: "LICENSE"},
{Type: "BSD-0-Clause", FilePath: "LICENSE.txt"},
{Type: "AGPL-3.0", FilePath: "foo/LICENSE"},
{Type: "MIT", FilePath: "foo/COPYING"},
},
want: true,
},
}
for _, test := range tests {
t.Run(test.label, func(t *testing.T) {
if got := licensesAreRedistributable(test.licenses); got != test.want {
t.Errorf("licensesAreRedistributable([licenses]) = %t, want %t", got, test.want)
}
})
}
}

52
internal/postgres/postgres.go
Просмотреть файл

@ -298,10 +298,10 @@ func (db *DB) GetPackage(ctx context.Context, path string, version string) (*int
}
var (
commitTime time.Time
name, synopsis, seriesPath, modulePath, suffix, readmeFilePath string
readmeContents []byte
licenseTypes, licensePaths []string
commitTime time.Time
name, synopsis, seriesPath, modulePath, suffix, readmeFilePath, versionType string
readmeContents []byte
licenseTypes, licensePaths []string
)
query := `
SELECT
@ -314,7 +314,8 @@ func (db *DB) GetPackage(ctx context.Context, path string, version string) (*int
v.module_path,
p.name,
p.synopsis,
p.suffix
p.suffix,
v.version_type
FROM
versions v
INNER JOIN
@ -333,7 +334,8 @@ func (db *DB) GetPackage(ctx context.Context, path string, version string) (*int
row := db.QueryRowContext(ctx, query, path, version)
if err := row.Scan(&commitTime, pq.Array(&licenseTypes),
pq.Array(&licensePaths), &readmeFilePath, &readmeContents, &seriesPath, &modulePath, &name, &synopsis, &suffix); err != nil {
pq.Array(&licensePaths), &readmeFilePath, &readmeContents, &seriesPath, &modulePath,
&name, &synopsis, &suffix, &versionType); err != nil {
if err == sql.ErrNoRows {
return nil, derrors.NotFound(fmt.Sprintf("package %s@%s not found", path, version))
}
@ -360,6 +362,7 @@ func (db *DB) GetPackage(ctx context.Context, path string, version string) (*int
CommitTime: commitTime,
ReadmeFilePath: readmeFilePath,
ReadmeContents: readmeContents,
VersionType: internal.VersionType(versionType),
},
}, nil
}
@ -797,6 +800,30 @@ func padPrerelease(v string) (string, error) {
return strings.Join(pre, "."), nil
}
// removeNonDistributableData removes any information from the version payload,
// after checking licenses.
func removeNonDistributableData(v *internal.Version) {
hasRedistributablePackage := false
for _, p := range v.Packages {
if p.IsRedistributable() {
hasRedistributablePackage = true
} else {
// Not redistributable, so prune information that can't be stored. In the
// future this should also include documentation.
p.Synopsis = ""
}
}
// If no packages are redistributable, we have no need for the readme
// contents, so drop them. Note that if a single package is redistributable,
// it must be true by definition that the module itself it redistributable,
// so capturing the README contents is OK.
if !hasRedistributablePackage {
v.ReadmeFilePath = ""
v.ReadmeContents = nil
}
}
// InsertVersion inserts a Version into the database along with any necessary
// series, modules and packages. If any of these rows already exist, they will
// not be updated. The version string is also parsed into major, minor, patch
@ -812,6 +839,8 @@ func (db *DB) InsertVersion(ctx context.Context, version *internal.Version, lice
return derrors.InvalidArgument(fmt.Sprintf("validateVersion: %v", err))
}
removeNonDistributableData(version)
return db.Transact(func(tx *sql.Tx) error {
if _, err := tx.ExecContext(ctx,
`INSERT INTO series(path)
@ -888,7 +917,15 @@ func (db *DB) InsertVersion(ctx context.Context, version *internal.Version, lice
var importValues []interface{}
var pkgLicenseValues []interface{}
for _, p := range version.Packages {
pkgValues = append(pkgValues, p.Path, p.Synopsis, p.Name, version.Version, version.ModulePath, p.Suffix)
pkgValues = append(pkgValues,
p.Path,
p.Synopsis,
p.Name,
version.Version,
version.ModulePath,
p.Suffix,
p.IsRedistributable(),
)
for _, l := range p.Licenses {
pkgLicenseValues = append(pkgLicenseValues, version.ModulePath, version.Version, l.FilePath, p.Path)
@ -906,6 +943,7 @@ func (db *DB) InsertVersion(ctx context.Context, version *internal.Version, lice
"version",
"module_path",
"suffix",
"redistributable",
}
table := "packages"
if err := bulkInsert(ctx, tx, table, pkgCols, pkgValues, true); err != nil {

5
internal/postgres/postgres_test.go
Просмотреть файл

@ -28,10 +28,10 @@ const (
var (
now = NowTruncated()
sampleLicenseInfos = []*internal.LicenseInfo{
{Type: "licensename", FilePath: "bar/LICENSE"},
{Type: "MIT", FilePath: "LICENSE"},
}
sampleLicenses = []*internal.License{
{LicenseInfo: *sampleLicenseInfos[0], Contents: []byte("Lorem Ipsum")},
{LicenseInfo: *sampleLicenseInfos[0], Contents: []byte(`Lorem Ipsum`)},
}
)
@ -51,6 +51,7 @@ func sampleVersion(mutators ...func(*internal.Version)) *internal.Version {
Name: "foo",
Synopsis: "This is a package synopsis",
Path: "path.to/foo",
Licenses: sampleLicenseInfos,
Imports: []*internal.Import{
&internal.Import{
Name: "bar",

6
internal/postgres/search_test.go
Просмотреть файл

@ -24,6 +24,7 @@ func TestInsertDocumentsAndSearch(t *testing.T) {
Name: "foo",
Path: "/path/to/foo",
Synopsis: "foo",
Licenses: sampleLicenseInfos,
},
}
})
@ -33,6 +34,7 @@ func TestInsertDocumentsAndSearch(t *testing.T) {
Name: "bar",
Path: "/path/to/bar",
Synopsis: "bar is bar", // add an extra 'bar' to make sorting deterministic
Licenses: sampleLicenseInfos,
},
}
})
@ -42,6 +44,7 @@ func TestInsertDocumentsAndSearch(t *testing.T) {
Name: "foo",
Path: "/path/to/foo",
Synopsis: "foo",
Licenses: sampleLicenseInfos,
},
VersionInfo: internal.VersionInfo{
ModulePath: versionFoo.ModulePath,
@ -54,6 +57,7 @@ func TestInsertDocumentsAndSearch(t *testing.T) {
Name: "bar",
Path: "/path/to/bar",
Synopsis: "bar is bar",
Licenses: sampleLicenseInfos,
},
VersionInfo: internal.VersionInfo{
ModulePath: versionBar.ModulePath,
@ -160,7 +164,7 @@ func TestInsertDocumentsAndSearch(t *testing.T) {
if tc.versions != nil {
for _, v := range tc.versions {
if err := testDB.InsertVersion(ctx, v, nil); derrors.Type(err) != tc.insertErr {
if err := testDB.InsertVersion(ctx, v, sampleLicenses); derrors.Type(err) != tc.insertErr {
t.Fatalf("testDB.InsertVersion(%+v): %v", tc.versions, err)
}
if err := testDB.InsertDocuments(ctx, v); derrors.Type(err) != tc.insertErr {

1
internal/proxy/test_helper.go
Просмотреть файл

@ -54,6 +54,7 @@ func SetupTestProxy(ctx context.Context, t *testing.T) (func(t *testing.T), *Cli
for _, v := range [][]string{
[]string{"my.mod/module", "v1.0.0"},
[]string{"no.mod/module", "v1.0.0"},
[]string{"nonredistributable.mod/module", "v1.0.0"},
[]string{"emp.ty/module", "v1.0.0"},
[]string{"rsc.io/quote", "v1.5.2"},
[]string{"rsc.io/quote/v2", "v2.0.1"},

27
internal/proxy/testdata/modproxy/modules/nonredistributable.mod/module@v1.0.0/LICENSE поставляемый Normal file
Просмотреть файл

@ -0,0 +1,27 @@
Copyright (c) 2009 The Go Authors. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

5
internal/proxy/testdata/modproxy/modules/nonredistributable.mod/module@v1.0.0/LICENSE.txt поставляемый Normal file
Просмотреть файл

@ -0,0 +1,5 @@
Copyright 2019 Google Inc
Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

1
internal/proxy/testdata/modproxy/modules/nonredistributable.mod/module@v1.0.0/README.md поставляемый Normal file
Просмотреть файл

@ -0,0 +1 @@
README FILE FOR TESTING.

7
internal/proxy/testdata/modproxy/modules/nonredistributable.mod/module@v1.0.0/bar/LICENSE поставляемый Normal file
Просмотреть файл

@ -0,0 +1,7 @@
Copyright 2019 Google Inc
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

11
internal/proxy/testdata/modproxy/modules/nonredistributable.mod/module@v1.0.0/bar/bar.go поставляемый Normal file
Просмотреть файл

@ -0,0 +1,11 @@
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// package bar
package bar
// Bar returns the string "bar".
func Bar() string {
return "bar"
}

7
internal/proxy/testdata/modproxy/modules/nonredistributable.mod/module@v1.0.0/bar/baz/COPYING поставляемый Normal file
Просмотреть файл

@ -0,0 +1,7 @@
Copyright 2019 Google Inc
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

5
internal/proxy/testdata/modproxy/modules/nonredistributable.mod/module@v1.0.0/bar/baz/LICENSE поставляемый Normal file
Просмотреть файл

@ -0,0 +1,5 @@
Copyright 2019 Google Inc
Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

11
internal/proxy/testdata/modproxy/modules/nonredistributable.mod/module@v1.0.0/bar/baz/baz.go поставляемый Normal file
Просмотреть файл

@ -0,0 +1,11 @@
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// package baz
package baz
// Baz returns the string "baz".
func Baz() string {
return "baz"
}

5
internal/proxy/testdata/modproxy/modules/nonredistributable.mod/module@v1.0.0/foo/LICENSE.md поставляемый Normal file
Просмотреть файл

@ -0,0 +1,5 @@
Copyright 2019 Google Inc
Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

17
internal/proxy/testdata/modproxy/modules/nonredistributable.mod/module@v1.0.0/foo/foo.go поставляемый Normal file
Просмотреть файл

@ -0,0 +1,17 @@
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// package foo
package foo
import (
"fmt"
"my.mod/module/bar"
)
// FooBar returns the string "foo bar".
func FooBar() string {
return fmt.Sprintf("foo %s", bar.Bar())
}

3
internal/proxy/testdata/modproxy/modules/nonredistributable.mod/module@v1.0.0/go.mod поставляемый Normal file
Просмотреть файл

@ -0,0 +1,3 @@
module nonredistributable.mod/module
go 1.12

3
internal/proxy/testdata/modproxy/proxy/nonredistributable.mod/module/@v/list поставляемый Normal file
Просмотреть файл

@ -0,0 +1,3 @@
v1.0.0
v1.1.0
v1.0.1

4
internal/proxy/testdata/modproxy/proxy/nonredistributable.mod/module/@v/v1.0.0.info поставляемый Normal file
Просмотреть файл

@ -0,0 +1,4 @@
{
"Version": "v1.0.0",
"Time": "2019-01-30T00:00:00Z"
}

1
internal/proxy/testdata/modproxy/proxy/nonredistributable.mod/module/@v/v1.0.0.mod поставляемый Normal file
Просмотреть файл

@ -0,0 +1 @@
module nonredistributable.mod/module

4
internal/proxy/testdata/modproxy/proxy/nonredistributable.mod/module/@v/v1.1.0.info поставляемый Normal file
Просмотреть файл

@ -0,0 +1,4 @@
{
"Version": "v1.1.0",
"Time": "2019-02-01T00:00:00Z"
}

1
internal/proxy/testdata/modproxy/proxy/nonredistributable.mod/module/@v/v1.1.0.mod поставляемый Normal file
Просмотреть файл

@ -0,0 +1 @@
module nonredistributable.mod/module

4
internal/proxy/testdata/modproxy/proxy/nonredistributable.mod/module/@v/v1.1.1.info поставляемый Normal file
Просмотреть файл

@ -0,0 +1,4 @@
{
"Version": "v1.1.1",
"Time": "2019-02-06T00:00:00Z"
}

1
internal/proxy/testdata/modproxy/proxy/nonredistributable.mod/module/@v/v1.1.1.mod поставляемый Normal file
Просмотреть файл

@ -0,0 +1 @@
module nonredistributable.mod/module