access.yaml support (#204)

- Added support to load git access tokens from environment variables

cmd/install.go

@@ -39,7 +39,16 @@ func Install(path string) (err error) {
 			generator = &generators.HelmGenerator{}
 		}
 
-		if err := component.Install(path, generator); err != nil {
+		// Attempt to load access tokens if we are on the root component
+		accessTokens := map[string]string{}
+		if path == "./" {
+			accessTokens, err = component.GetAccessTokens()
+			if err != nil {
+				return err
+			}
+		}
+
+		if err := component.Install(path, generator, accessTokens); err != nil {
 			return err
 		}
 

cmd/install_test.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+	"os"
 	"os/exec"
 	"testing"
 
@@ -33,3 +34,25 @@ func TestInstallWithHooks(t *testing.T) {
 
 	assert.Nil(t, err)
 }
+
+func TestInstallPrivateComponent(t *testing.T) {
+	cwd, err := os.Getwd()
+	assert.Nil(t, err)
+	defer func() {
+		_ = os.Chdir(cwd)
+	}()
+
+	// Change cwd to component directory
+	assert.Nil(t, os.Chdir("../test/fixtures/install-private"))
+
+	// Should fail with no environment var set to personal_access_token
+	assert.NotNil(t, Install("./"))
+	assert.Nil(t, os.Chdir("./"))
+
+	// If a personal access token exists, assume its correct and Install should succeed
+	if _, exists := os.LookupEnv("personal_access_token"); exists {
+		assert.Nil(t, Install("./"))
+	} else {
+		assert.NotNil(t, Install("./"))
+	}
+}

cmd/set.go

@@ -119,7 +119,7 @@ func Set(environment string, subcomponent string, pathValuePairStrings []string,
 		return err
 	}
 
-	newConfigError := errors.New("New configuration was specified and the --no-new-config-keys switch is on.")
+	newConfigError := errors.New("new configuration was specified and the --no-new-config-keys switch is on")
 
 	for _, pathValue := range pathValuePairs {
 		if noNewConfigKeys {

cmd/version.go

@@ -30,7 +30,7 @@ var versionCmd = &cobra.Command{
 
 // PrintVersion prints the current version of Fabrikate being used.
 func PrintVersion() {
-	log.Println("fab version 0.12.0")
+	log.Println("fab version 0.13.0")
 }
 
 func init() {

core/component.go

@@ -188,7 +188,7 @@ func (c *Component) AfterInstall() (err error) {
 }
 
 // InstallComponent installs the component (if needed) utilizing its Method.
-func (c *Component) InstallComponent(componentPath string) (err error) {
+func (c *Component) InstallComponent(componentPath string, accessTokens map[string]string) (err error) {
 	if c.Method == "git" {
 		componentsPath := fmt.Sprintf("%s/components", componentPath)
 		if err := exec.Command("mkdir", "-p", componentsPath).Run(); err != nil {
@@ -201,7 +201,13 @@ func (c *Component) InstallComponent(componentPath string) (err error) {
 		}
 
 		log.Println(emoji.Sprintf(":helicopter: Installing component %s with git from %s", c.Name, c.Source))
-		if err = CloneRepo(c.Source, c.Version, subcomponentPath, c.Branch); err != nil {
+
+		accessToken := ""
+		if foundToken, ok := accessTokens[c.Source]; ok {
+			accessToken = foundToken
+		}
+
+		if err = CloneRepo(c.Source, c.Version, subcomponentPath, c.Branch, accessToken); err != nil {
 			return err
 		}
 	}
@@ -211,19 +217,19 @@ func (c *Component) InstallComponent(componentPath string) (err error) {
 
 // Install encapsulates the install lifecycle of a component including before-install,
 // installation, and after-install hooks.
-func (c *Component) Install(componentPath string, generator Generator) (err error) {
+func (c *Component) Install(componentPath string, generator Generator, accessTokens map[string]string) (err error) {
 	if err := c.BeforeInstall(); err != nil {
 		return err
 	}
 
 	for _, subcomponent := range c.Subcomponents {
-		if err := subcomponent.InstallComponent(componentPath); err != nil {
+		if err := subcomponent.InstallComponent(componentPath, accessTokens); err != nil {
 			return err
 		}
 	}
 
 	if generator != nil {
-		if err := generator.Install(c); err != nil {
+		if err := generator.Install(c, accessTokens); err != nil {
 			return err
 		}
 	}
@@ -462,3 +468,24 @@ func (c *Component) sortSubcomponents() {
 		return c.Subcomponents[i].Name < c.Subcomponents[j].Name
 	})
 }
+
+// GetAccessTokens attempts to find an access.yaml file in the same physical directory of the component.
+// Un-marshalling if found,
+func (c *Component) GetAccessTokens() (tokens map[string]string, err error) {
+	// If access.yaml is found in same directory of component.yaml, see if c.Source is in the map and use the value as accessToken
+	accessYamlPath := path.Join(c.PhysicalPath, "access.yaml")
+	err = UnmarshalFile(accessYamlPath, yaml.Unmarshal, &tokens)
+	if os.IsNotExist(err) {
+		// If the file is not found, return an empty map with no error
+		return map[string]string{}, nil
+	}
+	for repo, envVar := range tokens {
+		token := os.Getenv(envVar)
+		if token == "" {
+			log.Error(emoji.Sprintf(":no_entry_sign: attempted to load environment variable %s; but is either not set or an empty string.", envVar))
+		} else {
+			tokens[repo] = token
+		}
+	}
+	return tokens, err
+}

core/generator.go

@@ -4,5 +4,5 @@ package core
 // to install and generate resource manifests.
 type Generator interface {
 	Generate(component *Component) (manifest string, err error)
-	Install(component *Component) (err error)
+	Install(component *Component, accessTokens map[string]string) (err error)
 }

core/git.go

@@ -1,14 +1,33 @@
 package core
 
 import (
+	"fmt"
 	"os/exec"
+	"regexp"
 
 	"github.com/kyokomi/emoji"
 	log "github.com/sirupsen/logrus"
 )
 
-// CloneRepo is a helper func to centralize cloning a repo with the spec provided by its arguments.
-func CloneRepo(repo string, commit string, intoPath string, branch string) (err error) {
+// CloneRepo is a helper func to centralize cloning a repository with the spec provided by its arguments.
+func CloneRepo(repo string, commit string, intoPath string, branch string, accessToken string) (err error) {
+	if accessToken != "" {
+		// Only match when the repo string does not contain a an access token already
+		// "(https?)://(?!(.+:)?.+@)(.+)" would be preferred but go does not support negative lookahead
+		pattern, err := regexp.Compile("^(https?)://([^@]+@)?(.+)$")
+		if err != nil {
+			return err
+		}
+		// If match is found, inject the access token into the repo string
+		matches := pattern.FindStringSubmatch(repo)
+		if matches != nil {
+			protocol := matches[1]
+			// credentialsWithAtSign := matches[2]
+			cleanedRepoString := matches[3]
+			repo = fmt.Sprintf("%v://%v@%v", protocol, accessToken, cleanedRepoString)
+		}
+	}
+
 	cloneArgs := []string{
 		"clone",
 		repo,
@@ -28,8 +47,9 @@ func CloneRepo(repo string, commit string, intoPath string, branch string) (err
 	}
 
 	cloneArgs = append(cloneArgs, intoPath)
-
-	if err = exec.Command("git", cloneArgs...).Run(); err != nil {
+	cloneCommand := exec.Command("git", cloneArgs...)
+	cloneCommand.Env = append(cloneCommand.Env, "GIT_TERMINAL_PROMPT=0") // tell git to fail if it asks for credentials
+	if err = cloneCommand.Run(); err != nil {
 		return err
 	}
 

generators/helm.go

@@ -124,7 +124,7 @@ func (hg *HelmGenerator) Generate(component *core.Component) (manifest string, e
 
 // Install installs the helm chart specified by the passed component and performs any
 // helm lifecycle events needed.
-func (hg *HelmGenerator) Install(component *core.Component) (err error) {
+func (hg *HelmGenerator) Install(component *core.Component, accessTokens map[string]string) (err error) {
 	if len(component.Source) == 0 || component.Method != "git" {
 		return nil
 	}
@@ -139,7 +139,14 @@ func (hg *HelmGenerator) Install(component *core.Component) (err error) {
 	}
 
 	log.Println(emoji.Sprintf(":helicopter: Installing helm repo %s for %s into %s", component.Source, component.Name, helmRepoPath))
-	if err = core.CloneRepo(component.Source, component.Version, helmRepoPath, component.Branch); err != nil {
+
+	// Access token lookup
+	accessToken := ""
+	if foundToken, ok := accessTokens[component.Source]; ok {
+		accessToken = foundToken
+	}
+
+	if err = core.CloneRepo(component.Source, component.Version, helmRepoPath, component.Branch, accessToken); err != nil {
 		return err
 	}
 

generators/static.go

@@ -37,6 +37,6 @@ func (sg *StaticGenerator) Generate(component *core.Component) (manifest string,
 
 // Install is provided such that the StaticGenerator fulfills the Generator interface.
 // Currently is a no-op, but could be extended to support remote static content (see #155)
-func (sg *StaticGenerator) Install(component *core.Component) (err error) {
+func (sg *StaticGenerator) Install(component *core.Component, accessTokens map[string]string) (err error) {
 	return nil
 }

test/fixtures/add/component.yaml

@@ -1,11 +1,12 @@
 name: add
+type: component
 subcomponents:
 - name: cloud-native
-  generator: component
+  type: component
   source: https://github.com/timfpark/fabrikate-cloud-native
   method: git
 - name: elasticsearch
-  generator: helm
+  type: helm
   source: https://github.com/helm/charts
   method: git
   path: stable/elasticsearch

test/fixtures/install-private/access.yaml

@@ -0,0 +1 @@
+https://github.com/evanlouie/fabrikate-private-component: personal_access_token

test/fixtures/install-private/component.yaml

@@ -0,0 +1,5 @@
+name: foobar
+subcomponents:
+  - name: my-private-repo
+    source: https://github.com/evanlouie/fabrikate-private-component
+    method: git

test/fixtures/install/component.json

@@ -1,5 +1,6 @@
 {
   "name": "microservices-workload",
+  "type": "component",
   "subcomponents": [
     {
       "name": "infra",