From 22ccbbaae82ca62d7b25f2c5e4f2fd280d255801 Mon Sep 17 00:00:00 2001 From: Owen Mansel-Chan Date: Tue, 3 May 2022 14:57:13 +0100 Subject: [PATCH 1/3] Run `go mod tidy -e` if go.mod exists --- .../cli/go-autobuilder/go-autobuilder.go | 74 ++++++++++++++++++- 1 file changed, 73 insertions(+), 1 deletion(-) diff --git a/extractor/cli/go-autobuilder/go-autobuilder.go b/extractor/cli/go-autobuilder/go-autobuilder.go index daaf733f..d37e1293 100644 --- a/extractor/cli/go-autobuilder/go-autobuilder.go +++ b/extractor/cli/go-autobuilder/go-autobuilder.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "golang.org/x/mod/semver" "io/ioutil" "log" "net/url" @@ -13,6 +12,8 @@ import ( "runtime" "strings" + "golang.org/x/mod/semver" + "github.com/github/codeql-go/extractor/autobuilder" "github.com/github/codeql-go/extractor/util" ) @@ -289,6 +290,77 @@ func main() { } } + if depMode == GoGetWithModules { + // stat go.mod and go.sum + var beforeGoModFileInfo, beforeGoSumFileInfo os.FileInfo + + beforeGoMod, beforeGoModerr := os.Open("go.mod") + if beforeGoModerr == nil { + var beforeGoModStatErr error + beforeGoModFileInfo, beforeGoModStatErr = beforeGoMod.Stat() + if beforeGoModStatErr != nil { + log.Println("Failed to stat go.mod before running `go mod tidy -e`") + } + } else { + log.Println("Failed to read go.mod before running `go mod tidy -e`") + } + beforeGoMod.Close() + + beforeGoSum, beforeGoSumErr := os.Open("go.sum") + if beforeGoSumErr == nil { + var beforeGoSumStatErr error + beforeGoSumFileInfo, beforeGoSumStatErr = beforeGoSum.Stat() + if beforeGoSumStatErr != nil { + log.Println("Failed to stat go.sum before running `go mod tidy -e`") + } + } + // don't print a warning if beforeGoSumErr != nil as it may be that the + // file doesn't exist + beforeGoSum.Close() + + // run `go mod tidy -e` + res := util.RunCmd(exec.Command("go", "mod", "tidy", "-e")) + + if !res { + log.Println("Failed to run `go mod tidy -e`") + } else { + if beforeGoModFileInfo != nil { + afterGoMod, afterGoModErr := os.Open("go.mod") + if afterGoModErr != nil { + log.Println("Failed to read go.mod after running `go mod tidy -e`") + } else { + afterGoModFileInfo, afterGoModStatErr := afterGoMod.Stat() + if afterGoModStatErr != nil { + log.Println("Failed to stat go.mod after running `go mod tidy -e`") + } else { + if afterGoModFileInfo.ModTime().After(beforeGoModFileInfo.ModTime()) { + // if go.mod has been changed then notify the user + log.Println("We have run `go mod tidy -e` and it altered go.mod. You may wish to check these changes into version control. ") + } + } + } + afterGoMod.Close() + } + + afterGoSum, afterGoSumErr := os.Open("go.sum") + if afterGoSumErr != nil { + log.Println("Failed to read go.sum after running `go mod tidy -e`") + } else { + afterGoSumFileInfo, afterGoSumStatErr := afterGoSum.Stat() + if afterGoSumStatErr != nil { + log.Println("Failed to stat go.sum after running `go mod tidy -e`") + } else { + if beforeGoSumErr != nil || afterGoSumFileInfo.ModTime().After(beforeGoSumFileInfo.ModTime()) { + // if go.sum has been changed then notify the user + log.Println("We have run `go mod tidy -e` and it altered go.sum. You may wish to check these changes into version control. ") + } + } + } + afterGoSum.Close() + + } + } + // if `LGTM_INDEX_NEED_GOPATH` is set, it overrides the value for `needGopath` inferred above if needGopathOverride := os.Getenv("LGTM_INDEX_NEED_GOPATH"); needGopathOverride != "" { inLGTM = true From 570d3f47c491c40b21b3b28013eeb51b5c114140 Mon Sep 17 00:00:00 2001 From: Owen Mansel-Chan Date: Wed, 4 May 2022 10:11:53 +0100 Subject: [PATCH 2/3] Use `os.Stat` instead of `os.File.Stat` --- .../cli/go-autobuilder/go-autobuilder.go | 61 +++++-------------- 1 file changed, 14 insertions(+), 47 deletions(-) diff --git a/extractor/cli/go-autobuilder/go-autobuilder.go b/extractor/cli/go-autobuilder/go-autobuilder.go index d37e1293..681bf666 100644 --- a/extractor/cli/go-autobuilder/go-autobuilder.go +++ b/extractor/cli/go-autobuilder/go-autobuilder.go @@ -292,31 +292,12 @@ func main() { if depMode == GoGetWithModules { // stat go.mod and go.sum - var beforeGoModFileInfo, beforeGoSumFileInfo os.FileInfo - - beforeGoMod, beforeGoModerr := os.Open("go.mod") - if beforeGoModerr == nil { - var beforeGoModStatErr error - beforeGoModFileInfo, beforeGoModStatErr = beforeGoMod.Stat() - if beforeGoModStatErr != nil { - log.Println("Failed to stat go.mod before running `go mod tidy -e`") - } - } else { - log.Println("Failed to read go.mod before running `go mod tidy -e`") + beforeGoModFileInfo, beforeGoModErr := os.Stat("go.mod") + if beforeGoModErr != nil { + log.Println("Failed to stat go.mod before running `go mod tidy -e`") } - beforeGoMod.Close() - beforeGoSum, beforeGoSumErr := os.Open("go.sum") - if beforeGoSumErr == nil { - var beforeGoSumStatErr error - beforeGoSumFileInfo, beforeGoSumStatErr = beforeGoSum.Stat() - if beforeGoSumStatErr != nil { - log.Println("Failed to stat go.sum before running `go mod tidy -e`") - } - } - // don't print a warning if beforeGoSumErr != nil as it may be that the - // file doesn't exist - beforeGoSum.Close() + beforeGoSumFileInfo, beforeGoSumErr := os.Stat("go.sum") // run `go mod tidy -e` res := util.RunCmd(exec.Command("go", "mod", "tidy", "-e")) @@ -325,38 +306,24 @@ func main() { log.Println("Failed to run `go mod tidy -e`") } else { if beforeGoModFileInfo != nil { - afterGoMod, afterGoModErr := os.Open("go.mod") + afterGoModFileInfo, afterGoModErr := os.Stat("go.mod") if afterGoModErr != nil { - log.Println("Failed to read go.mod after running `go mod tidy -e`") - } else { - afterGoModFileInfo, afterGoModStatErr := afterGoMod.Stat() - if afterGoModStatErr != nil { - log.Println("Failed to stat go.mod after running `go mod tidy -e`") - } else { - if afterGoModFileInfo.ModTime().After(beforeGoModFileInfo.ModTime()) { - // if go.mod has been changed then notify the user - log.Println("We have run `go mod tidy -e` and it altered go.mod. You may wish to check these changes into version control. ") - } - } + log.Println("Failed to stat go.mod after running `go mod tidy -e`") + } else if afterGoModFileInfo.ModTime().After(beforeGoModFileInfo.ModTime()) { + // if go.mod has been changed then notify the user + log.Println("We have run `go mod tidy -e` and it altered go.mod. You may wish to check these changes into version control. ") } - afterGoMod.Close() } - afterGoSum, afterGoSumErr := os.Open("go.sum") + afterGoSumFileInfo, afterGoSumErr := os.Stat("go.sum") if afterGoSumErr != nil { - log.Println("Failed to read go.sum after running `go mod tidy -e`") + log.Println("Failed to stat go.sum after running `go mod tidy -e`") } else { - afterGoSumFileInfo, afterGoSumStatErr := afterGoSum.Stat() - if afterGoSumStatErr != nil { - log.Println("Failed to stat go.sum after running `go mod tidy -e`") - } else { - if beforeGoSumErr != nil || afterGoSumFileInfo.ModTime().After(beforeGoSumFileInfo.ModTime()) { - // if go.sum has been changed then notify the user - log.Println("We have run `go mod tidy -e` and it altered go.sum. You may wish to check these changes into version control. ") - } + if beforeGoSumErr != nil || afterGoSumFileInfo.ModTime().After(beforeGoSumFileInfo.ModTime()) { + // if go.sum has been changed then notify the user + log.Println("We have run `go mod tidy -e` and it altered go.sum. You may wish to check these changes into version control. ") } } - afterGoSum.Close() } } From 2930bd4cc24489be4a915a6e759e1872a8e81ef3 Mon Sep 17 00:00:00 2001 From: Chris Smowton Date: Fri, 13 May 2022 17:32:00 +0100 Subject: [PATCH 3/3] Only attempt go.mod updating if go >= 1.16 Prior to this (a) Go will attempt to update go.mod/sum anyhow, and (b) the `mod tidy -e` option isn't available. --- extractor/cli/go-autobuilder/go-autobuilder.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extractor/cli/go-autobuilder/go-autobuilder.go b/extractor/cli/go-autobuilder/go-autobuilder.go index 681bf666..07b8a6cb 100644 --- a/extractor/cli/go-autobuilder/go-autobuilder.go +++ b/extractor/cli/go-autobuilder/go-autobuilder.go @@ -290,7 +290,8 @@ func main() { } } - if depMode == GoGetWithModules { + // Go 1.16 and later won't automatically attempt to update go.mod / go.sum during package loading, so try to update them here: + if depMode == GoGetWithModules && semver.Compare(getEnvGoSemVer(), "1.16") >= 0 { // stat go.mod and go.sum beforeGoModFileInfo, beforeGoModErr := os.Stat("go.mod") if beforeGoModErr != nil { @@ -324,7 +325,6 @@ func main() { log.Println("We have run `go mod tidy -e` and it altered go.sum. You may wish to check these changes into version control. ") } } - } }