зеркало из https://github.com/CryptoPro/go.git
cmd/go: have go mod vendor copy embedded files in subdirs
If a package vendored with go mod vendor depends on embedded files contained in subdirectories, copy them into the the corresponding place in the module's vendor tree. (Embeds in parent directories are disallowed by the embed pattern rules, and embeds in the same directory are copied because go mod vendor already copies the non-go files in the package's own directory). Export the vendor pattern expansion code in internal/load so internal/modcmd's vendor code can use it. Fixes #43077 Change-Id: I61edb344d73df590574a6498ffb6069e8d72a147 Reviewed-on: https://go-review.googlesource.com/c/go/+/283641 Trust: Michael Matloob <matloob@golang.org> Trust: Bryan C. Mills <bcmills@google.com> Run-TryBot: Michael Matloob <matloob@golang.org> TryBot-Result: Go Bot <gobot@golang.org> Reviewed-by: Bryan C. Mills <bcmills@google.com> Reviewed-by: Jay Conrod <jayconrod@google.com>
This commit is contained in:
Michael Matloob
Родитель
be28e5abc5
Коммит
6e243ce71d
|
|
@ -581,8 +581,6 @@ func runList(ctx context.Context, cmd *base.Command, args []string) {
|
|||
// Show vendor-expanded paths in listing
|
||||
p.TestImports = p.Resolve(p.TestImports)
|
||||
p.XTestImports = p.Resolve(p.XTestImports)
|
||||
p.TestEmbedFiles = p.ResolveEmbed(p.TestEmbedPatterns)
|
||||
p.XTestEmbedFiles = p.ResolveEmbed(p.XTestEmbedPatterns)
|
||||
p.DepOnly = !cmdline[p]
|
||||
|
||||
if *listCompiled {
|
||||
|
|
|
|||
|
|
@ -1807,7 +1807,7 @@ func (p *Package) load(ctx context.Context, path string, stk *ImportStack, impor
|
|||
stk.Push(path)
|
||||
defer stk.Pop()
|
||||
|
||||
p.EmbedFiles, p.Internal.Embed, err = p.resolveEmbed(p.EmbedPatterns)
|
||||
p.EmbedFiles, p.Internal.Embed, err = resolveEmbed(p.Dir, p.EmbedPatterns)
|
||||
if err != nil {
|
||||
setError(err)
|
||||
embedErr := err.(*EmbedError)
|
||||
|
|
@ -1932,17 +1932,20 @@ func (e *EmbedError) Unwrap() error {
|
|||
}
|
||||
|
||||
// ResolveEmbed resolves //go:embed patterns and returns only the file list.
|
||||
// For use by go list to compute p.TestEmbedFiles and p.XTestEmbedFiles.
|
||||
func (p *Package) ResolveEmbed(patterns []string) []string {
|
||||
files, _, _ := p.resolveEmbed(patterns)
|
||||
return files
|
||||
// For use by go mod vendor to find embedded files it should copy into the
|
||||
// vendor directory.
|
||||
// TODO(#42504): Once go mod vendor uses load.PackagesAndErrors, just
|
||||
// call (*Package).ResolveEmbed
|
||||
func ResolveEmbed(dir string, patterns []string) ([]string, error) {
|
||||
files, _, err := resolveEmbed(dir, patterns)
|
||||
return files, err
|
||||
}
|
||||
|
||||
// resolveEmbed resolves //go:embed patterns to precise file lists.
|
||||
// It sets files to the list of unique files matched (for go list),
|
||||
// and it sets pmap to the more precise mapping from
|
||||
// patterns to files.
|
||||
func (p *Package) resolveEmbed(patterns []string) (files []string, pmap map[string][]string, err error) {
|
||||
func resolveEmbed(pkgdir string, patterns []string) (files []string, pmap map[string][]string, err error) {
|
||||
var pattern string
|
||||
defer func() {
|
||||
if err != nil {
|
||||
|
|
@ -1953,6 +1956,7 @@ func (p *Package) resolveEmbed(patterns []string) (files []string, pmap map[stri
|
|||
}
|
||||
}()
|
||||
|
||||
// TODO(rsc): All these messages need position information for better error reports.
|
||||
pmap = make(map[string][]string)
|
||||
have := make(map[string]int)
|
||||
dirOK := make(map[string]bool)
|
||||
|
|
@ -1966,7 +1970,7 @@ func (p *Package) resolveEmbed(patterns []string) (files []string, pmap map[stri
|
|||
}
|
||||
|
||||
// Glob to find matches.
|
||||
match, err := fsys.Glob(p.Dir + string(filepath.Separator) + filepath.FromSlash(pattern))
|
||||
match, err := fsys.Glob(pkgdir + string(filepath.Separator) + filepath.FromSlash(pattern))
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
|
@ -1977,7 +1981,7 @@ func (p *Package) resolveEmbed(patterns []string) (files []string, pmap map[stri
|
|||
// then there may be other things lying around, like symbolic links or .git directories.)
|
||||
var list []string
|
||||
for _, file := range match {
|
||||
rel := filepath.ToSlash(file[len(p.Dir)+1:]) // file, relative to p.Dir
|
||||
rel := filepath.ToSlash(file[len(pkgdir)+1:]) // file, relative to p.Dir
|
||||
|
||||
what := "file"
|
||||
info, err := fsys.Lstat(file)
|
||||
|
|
@ -1990,13 +1994,13 @@ func (p *Package) resolveEmbed(patterns []string) (files []string, pmap map[stri
|
|||
|
||||
// Check that directories along path do not begin a new module
|
||||
// (do not contain a go.mod).
|
||||
for dir := file; len(dir) > len(p.Dir)+1 && !dirOK[dir]; dir = filepath.Dir(dir) {
|
||||
for dir := file; len(dir) > len(pkgdir)+1 && !dirOK[dir]; dir = filepath.Dir(dir) {
|
||||
if _, err := fsys.Stat(filepath.Join(dir, "go.mod")); err == nil {
|
||||
return nil, nil, fmt.Errorf("cannot embed %s %s: in different module", what, rel)
|
||||
}
|
||||
if dir != file {
|
||||
if info, err := fsys.Lstat(dir); err == nil && !info.IsDir() {
|
||||
return nil, nil, fmt.Errorf("cannot embed %s %s: in non-directory %s", what, rel, dir[len(p.Dir)+1:])
|
||||
return nil, nil, fmt.Errorf("cannot embed %s %s: in non-directory %s", what, rel, dir[len(pkgdir)+1:])
|
||||
}
|
||||
}
|
||||
dirOK[dir] = true
|
||||
|
|
@ -2027,7 +2031,7 @@ func (p *Package) resolveEmbed(patterns []string) (files []string, pmap map[stri
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel := filepath.ToSlash(path[len(p.Dir)+1:])
|
||||
rel := filepath.ToSlash(path[len(pkgdir)+1:])
|
||||
name := info.Name()
|
||||
if path != file && (isBadEmbedName(name) || name[0] == '.' || name[0] == '_') {
|
||||
// Ignore bad names, assuming they won't go into modules.
|
||||
|
|
|
|||
|
|
@ -124,7 +124,7 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p
|
|||
imports = append(imports, p1)
|
||||
}
|
||||
var err error
|
||||
p.TestEmbedFiles, testEmbed, err = p.resolveEmbed(p.TestEmbedPatterns)
|
||||
p.TestEmbedFiles, testEmbed, err = resolveEmbed(p.Dir, p.TestEmbedPatterns)
|
||||
if err != nil && ptestErr == nil {
|
||||
ptestErr = &PackageError{
|
||||
ImportStack: stk.Copy(),
|
||||
|
|
@ -147,7 +147,7 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p
|
|||
}
|
||||
p.XTestImports[i] = p1.ImportPath
|
||||
}
|
||||
p.XTestEmbedFiles, xtestEmbed, err = p.resolveEmbed(p.XTestEmbedPatterns)
|
||||
p.XTestEmbedFiles, xtestEmbed, err = resolveEmbed(p.Dir, p.XTestEmbedPatterns)
|
||||
if err != nil && pxtestErr == nil {
|
||||
pxtestErr = &PackageError{
|
||||
ImportStack: stk.Copy(),
|
||||
|
|
|
|||
|
|
@ -7,7 +7,9 @@ package modcmd
|
|||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"go/build"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
|
|
@ -19,7 +21,9 @@ import (
|
|||
"cmd/go/internal/cfg"
|
||||
"cmd/go/internal/fsys"
|
||||
"cmd/go/internal/imports"
|
||||
"cmd/go/internal/load"
|
||||
"cmd/go/internal/modload"
|
||||
"cmd/go/internal/str"
|
||||
|
||||
"golang.org/x/mod/module"
|
||||
"golang.org/x/mod/semver"
|
||||
|
|
@ -182,19 +186,76 @@ func moduleLine(m, r module.Version) string {
|
|||
}
|
||||
|
||||
func vendorPkg(vdir, pkg string) {
|
||||
// TODO(#42504): Instead of calling modload.ImportMap then build.ImportDir,
|
||||
// just call load.PackagesAndErrors. To do that, we need to add a good way
|
||||
// to ignore build constraints.
|
||||
realPath := modload.ImportMap(pkg)
|
||||
if realPath != pkg && modload.ImportMap(realPath) != "" {
|
||||
fmt.Fprintf(os.Stderr, "warning: %s imported as both %s and %s; making two copies.\n", realPath, realPath, pkg)
|
||||
}
|
||||
|
||||
copiedFiles := make(map[string]bool)
|
||||
dst := filepath.Join(vdir, pkg)
|
||||
src := modload.PackageDir(realPath)
|
||||
if src == "" {
|
||||
fmt.Fprintf(os.Stderr, "internal error: no pkg for %s -> %s\n", pkg, realPath)
|
||||
}
|
||||
copyDir(dst, src, matchPotentialSourceFile)
|
||||
copyDir(dst, src, matchPotentialSourceFile, copiedFiles)
|
||||
if m := modload.PackageModule(realPath); m.Path != "" {
|
||||
copyMetadata(m.Path, realPath, dst, src)
|
||||
copyMetadata(m.Path, realPath, dst, src, copiedFiles)
|
||||
}
|
||||
|
||||
ctx := build.Default
|
||||
ctx.UseAllFiles = true
|
||||
bp, err := ctx.ImportDir(src, build.IgnoreVendor)
|
||||
// Because UseAllFiles is set on the build.Context, it's possible ta get
|
||||
// a MultiplePackageError on an otherwise valid package: the package could
|
||||
// have different names for GOOS=windows and GOOS=mac for example. On the
|
||||
// other hand if there's a NoGoError, the package might have source files
|
||||
// specifying "// +build ignore" those packages should be skipped because
|
||||
// embeds from ignored files can't be used.
|
||||
// TODO(#42504): Find a better way to avoid errors from ImportDir. We'll
|
||||
// need to figure this out when we switch to PackagesAndErrors as per the
|
||||
// TODO above.
|
||||
var multiplePackageError *build.MultiplePackageError
|
||||
var noGoError *build.NoGoError
|
||||
if err != nil {
|
||||
if errors.As(err, &noGoError) {
|
||||
return // No source files in this package are built. Skip embeds in ignored files.
|
||||
} else if !errors.As(err, &multiplePackageError) { // multiplePackgeErrors are okay, but others are not.
|
||||
base.Fatalf("internal error: failed to find embedded files of %s: %v\n", pkg, err)
|
||||
}
|
||||
}
|
||||
embedPatterns := str.StringList(bp.EmbedPatterns, bp.TestEmbedPatterns, bp.XTestEmbedPatterns)
|
||||
embeds, err := load.ResolveEmbed(bp.Dir, embedPatterns)
|
||||
if err != nil {
|
||||
base.Fatalf("go mod vendor: %v", err)
|
||||
}
|
||||
for _, embed := range embeds {
|
||||
embedDst := filepath.Join(dst, embed)
|
||||
if copiedFiles[embedDst] {
|
||||
continue
|
||||
}
|
||||
|
||||
// Copy the file as is done by copyDir below.
|
||||
r, err := os.Open(filepath.Join(src, embed))
|
||||
if err != nil {
|
||||
base.Fatalf("go mod vendor: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(embedDst), 0777); err != nil {
|
||||
base.Fatalf("go mod vendor: %v", err)
|
||||
}
|
||||
w, err := os.Create(embedDst)
|
||||
if err != nil {
|
||||
base.Fatalf("go mod vendor: %v", err)
|
||||
}
|
||||
if _, err := io.Copy(w, r); err != nil {
|
||||
base.Fatalf("go mod vendor: %v", err)
|
||||
}
|
||||
r.Close()
|
||||
if err := w.Close(); err != nil {
|
||||
base.Fatalf("go mod vendor: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -207,14 +268,14 @@ var copiedMetadata = make(map[metakey]bool)
|
|||
|
||||
// copyMetadata copies metadata files from parents of src to parents of dst,
|
||||
// stopping after processing the src parent for modPath.
|
||||
func copyMetadata(modPath, pkg, dst, src string) {
|
||||
func copyMetadata(modPath, pkg, dst, src string, copiedFiles map[string]bool) {
|
||||
for parent := 0; ; parent++ {
|
||||
if copiedMetadata[metakey{modPath, dst}] {
|
||||
break
|
||||
}
|
||||
copiedMetadata[metakey{modPath, dst}] = true
|
||||
if parent > 0 {
|
||||
copyDir(dst, src, matchMetadata)
|
||||
copyDir(dst, src, matchMetadata, copiedFiles)
|
||||
}
|
||||
if modPath == pkg {
|
||||
break
|
||||
|
|
@ -282,7 +343,7 @@ func matchPotentialSourceFile(dir string, info fs.DirEntry) bool {
|
|||
}
|
||||
|
||||
// copyDir copies all regular files satisfying match(info) from src to dst.
|
||||
func copyDir(dst, src string, match func(dir string, info fs.DirEntry) bool) {
|
||||
func copyDir(dst, src string, match func(dir string, info fs.DirEntry) bool, copiedFiles map[string]bool) {
|
||||
files, err := os.ReadDir(src)
|
||||
if err != nil {
|
||||
base.Fatalf("go mod vendor: %v", err)
|
||||
|
|
@ -294,11 +355,14 @@ func copyDir(dst, src string, match func(dir string, info fs.DirEntry) bool) {
|
|||
if file.IsDir() || !file.Type().IsRegular() || !match(src, file) {
|
||||
continue
|
||||
}
|
||||
copiedFiles[file.Name()] = true
|
||||
r, err := os.Open(filepath.Join(src, file.Name()))
|
||||
if err != nil {
|
||||
base.Fatalf("go mod vendor: %v", err)
|
||||
}
|
||||
w, err := os.Create(filepath.Join(dst, file.Name()))
|
||||
dstPath := filepath.Join(dst, file.Name())
|
||||
copiedFiles[dstPath] = true
|
||||
w, err := os.Create(dstPath)
|
||||
if err != nil {
|
||||
base.Fatalf("go mod vendor: %v", err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,14 @@ go list -f '{{.EmbedPatterns}}'
|
|||
stdout '\[x\*t\*t\]'
|
||||
go list -f '{{.EmbedFiles}}'
|
||||
stdout '\[x.txt\]'
|
||||
go list -test -f '{{.TestEmbedPatterns}}'
|
||||
stdout '\[y\*t\*t\]'
|
||||
go list -test -f '{{.TestEmbedFiles}}'
|
||||
stdout '\[y.txt\]'
|
||||
go list -test -f '{{.XTestEmbedPatterns}}'
|
||||
stdout '\[z\*t\*t\]'
|
||||
go list -test -f '{{.XTestEmbedFiles}}'
|
||||
stdout '\[z.txt\]'
|
||||
|
||||
# build embeds x.txt
|
||||
go build -x
|
||||
|
|
@ -58,6 +66,22 @@ import "embed"
|
|||
//go:embed x*t*t
|
||||
var X embed.FS
|
||||
|
||||
-- x_test.go --
|
||||
package p
|
||||
|
||||
import "embed"
|
||||
|
||||
//go:embed y*t*t
|
||||
var Y string
|
||||
|
||||
-- x_x_test.go --
|
||||
package p_test
|
||||
|
||||
import "embed"
|
||||
|
||||
//go:embed z*t*t
|
||||
var Z string
|
||||
|
||||
-- x.go2 --
|
||||
package p
|
||||
|
||||
|
|
@ -69,6 +93,8 @@ var X embed.FS
|
|||
-- x.txt --
|
||||
hello
|
||||
|
||||
-- y.txt --
|
||||
-- z.txt --
|
||||
-- x.txt2 --
|
||||
not hello
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,179 @@
|
|||
go mod vendor
|
||||
cmp vendor/example.com/a/samedir_embed.txt a/samedir_embed.txt
|
||||
cmp vendor/example.com/a/subdir/embed.txt a/subdir/embed.txt
|
||||
cmp vendor/example.com/a/subdir/test/embed.txt a/subdir/test/embed.txt
|
||||
cmp vendor/example.com/a/subdir/test/xtest/embed.txt a/subdir/test/xtest/embed.txt
|
||||
|
||||
cd broken_no_matching_files
|
||||
! go mod vendor
|
||||
stderr 'go mod vendor: pattern foo.txt: no matching files found'
|
||||
|
||||
cd ../broken_bad_pattern
|
||||
! go mod vendor
|
||||
stderr 'go mod vendor: pattern ../foo.txt: invalid pattern syntax'
|
||||
|
||||
# matchPotentialSourceFile prunes out tests and unbuilt code.
|
||||
# Make sure that they are vendored if they are embedded files.
|
||||
cd ../embed_unbuilt
|
||||
go mod vendor
|
||||
cmp vendor/example.com/dep/unbuilt.go dep/unbuilt.go
|
||||
cmp vendor/example.com/dep/dep_test.go dep/dep_test.go
|
||||
! exists vendor/example.com/dep/not_embedded_unbuilt.go
|
||||
! exists vendor/example.com/dep/not_embedded_dep_test.go
|
||||
-- go.mod --
|
||||
module example.com/foo
|
||||
go 1.16
|
||||
|
||||
require (
|
||||
example.com/a v0.1.0
|
||||
)
|
||||
|
||||
replace (
|
||||
example.com/a v0.1.0 => ./a
|
||||
)
|
||||
-- foo.go --
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"example.com/a"
|
||||
)
|
||||
|
||||
func main() {
|
||||
fmt.Println(a.Str())
|
||||
}
|
||||
-- a/go.mod --
|
||||
module example.com/a
|
||||
-- a/a.go --
|
||||
package a
|
||||
|
||||
import _ "embed"
|
||||
|
||||
//go:embed samedir_embed.txt
|
||||
var sameDir string
|
||||
|
||||
//go:embed subdir/embed.txt
|
||||
var subDir string
|
||||
|
||||
func Str() string {
|
||||
return sameDir + subDir
|
||||
}
|
||||
-- a/a_test.go --
|
||||
package a
|
||||
|
||||
import _ "embed"
|
||||
|
||||
//go:embed subdir/test/embed.txt
|
||||
var subderTest string
|
||||
-- a/a_x_test.go --
|
||||
package a_test
|
||||
|
||||
import _ "embed"
|
||||
|
||||
//go:embed subdir/test/xtest/embed.txt
|
||||
var subdirXtest string
|
||||
-- a/samedir_embed.txt --
|
||||
embedded file in same directory as package
|
||||
-- a/subdir/embed.txt --
|
||||
embedded file in subdirectory of package
|
||||
-- a/subdir/test/embed.txt --
|
||||
embedded file of test in subdirectory of package
|
||||
-- a/subdir/test/xtest/embed.txt --
|
||||
embedded file of xtest in subdirectory of package
|
||||
-- broken_no_matching_files/go.mod --
|
||||
module example.com/broken
|
||||
go 1.16
|
||||
|
||||
require (
|
||||
example.com/brokendep v0.1.0
|
||||
)
|
||||
|
||||
replace (
|
||||
example.com/brokendep v0.1.0 => ./brokendep
|
||||
)
|
||||
-- broken_no_matching_files/f.go --
|
||||
package broken
|
||||
|
||||
import _ "example.com/brokendep"
|
||||
|
||||
func F() {}
|
||||
-- broken_no_matching_files/brokendep/go.mod --
|
||||
module example.com/brokendep
|
||||
go 1.16
|
||||
-- broken_no_matching_files/brokendep/f.go --
|
||||
package brokendep
|
||||
|
||||
import _ "embed"
|
||||
|
||||
//go:embed foo.txt
|
||||
var foo string
|
||||
-- broken_bad_pattern/go.mod --
|
||||
module example.com/broken
|
||||
go 1.16
|
||||
|
||||
require (
|
||||
example.com/brokendep v0.1.0
|
||||
)
|
||||
|
||||
replace (
|
||||
example.com/brokendep v0.1.0 => ./brokendep
|
||||
)
|
||||
-- broken_bad_pattern/f.go --
|
||||
package broken
|
||||
|
||||
import _ "example.com/brokendep"
|
||||
|
||||
func F() {}
|
||||
-- broken_bad_pattern/brokendep/go.mod --
|
||||
module example.com/brokendep
|
||||
go 1.16
|
||||
-- broken_bad_pattern/brokendep/f.go --
|
||||
package brokendep
|
||||
|
||||
import _ "embed"
|
||||
|
||||
//go:embed ../foo.txt
|
||||
var foo string
|
||||
-- embed_unbuilt/go.mod --
|
||||
module example.com/foo
|
||||
go 1.16
|
||||
|
||||
require (
|
||||
example.com/dep v0.1.0
|
||||
)
|
||||
|
||||
replace (
|
||||
example.com/dep v0.1.0 => ./dep
|
||||
)
|
||||
-- embed_unbuilt/foo.go --
|
||||
package a
|
||||
|
||||
import _ "example.com/dep"
|
||||
|
||||
func F() {}
|
||||
-- embed_unbuilt/dep/go.mod --
|
||||
module example.com/dep
|
||||
go 1.16
|
||||
-- embed_unbuilt/dep/dep.go --
|
||||
package dep
|
||||
|
||||
import _ "embed"
|
||||
|
||||
//go:embed unbuilt.go
|
||||
var unbuilt string
|
||||
|
||||
//go:embed dep_test.go
|
||||
var depTest string
|
||||
-- embed_unbuilt/dep/unbuilt.go --
|
||||
// +build ignore
|
||||
|
||||
package dep
|
||||
-- embed_unbuilt/dep/not_embedded_unbuilt.go --
|
||||
// +build ignore
|
||||
|
||||
package dep
|
||||
-- embed_unbuilt/dep/dep_test.go --
|
||||
package dep
|
||||
-- embed_unbuilt/dep/not_embedded_dep_test.go --
|
||||
package dep
|
||||
Загрузка…
Ссылка в новой задаче