Extend `--append` to remote backend, and the backend type now honors the existing type in append mode (#373)

* Extend `--append` to remote backend, and the backend type now honors the existing type in append mode

* Pass CI

* Update readme

README.md

@@ -191,23 +191,26 @@ After going through all the resources to be imported, users press `w` to instruc
 
 In non-interactive mode, `aztfexport` only imports the recognized resources, and skip the others. Users can further specify the `--continue`/`-k` option to make the tool continue even on hitting any import error.
 
-### Remote Backend
+### Backend: local vs remote
 
-By default `aztfexport` uses local backend to store the state file. While it is also possible to use [remote backend](https://www.terraform.io/language/settings/backends), via the `--backend-type` and `--backend-config` options.
+By default, `aztfexport` checks the output directory whether it is empty. If it is not empty, the user can specify either `--overwrite` to clean up the directory, or `--append` to additively generate the config to the directory.
 
-E.g. to use the [`azurerm` backend](https://www.terraform.io/language/settings/backends/azurerm#azurerm), users can invoke `aztfexport` like following:
+> 💡 In append mode, the file generated by `aztfexport` be named differently than normal, where each file will has `.aztfexport` suffix before the extension (e.g. `main.aztfexport.tf`), to avoid potential file name conflicts. If you run `aztfexport [subcommand] --append` multiple times, the generated config in `main.aztfexport.tf` will be appended in each run.
 
-```shell
-aztfexport [subcommand] --backend-type=azurerm --backend-config=resource_group_name=<resource group name> --backend-config=storage_account_name=<account name> --backend-config=container_name=<container name> --backend-config=key=terraform.tfstate 
-```
+On top of this, `aztfexport` supports importing these resources to state either in local backend or [remote backend](https://www.terraform.io/language/settings/backends):
 
-### Import Into Existing Local State
+- In case the output directory is empty, or the user has specified `--overwrite`, the backend type is determined by `--backend-type`, which defaults to `local` when absent.
+- Otherwise, the user append (via `--append`) to a non-empty output directory. Then `aztfexport` will honor any existing backend setting (i.e. in the [`terraform` setting](https://developer.hashicorp.com/terraform/language/settings)), and ensure it is consistent with the specified backend type (via `--backend-type`) and backend config (via `--backend-config`), if any.
 
-For local backend, `aztfexport` will by default ensure the output directory is empty at the very begining. This is to avoid any conflicts happen for existing user files, including the terraform configuration, provider configuration, the state file, etc. As a result, `aztfexport` generates a pretty new workspace for users.
+This means there are two ways to export into remote state:
 
-One limitation of doing so is users can't import resources to existing state file via `aztfexport`. To support this scenario, you can use the `--append` option. This option will make `aztfexport` skip the empty guarantee for the output directory. If the output directory is empty, then it has no effect. Otherwise, it will ensure the provider setting (create a file for it if not exists). Then it proceeds the following steps.
+- Using the `--backend-type` and `--backend-config`, e.g.:
 
-This means if the output directory has an active Terraform workspace, i.e. there exists a state file, any resource imported by the `aztfexport` will be imported into that state file. Especially, the file generated by `aztfexport` in this case will be named differently than normal, where each file will has `.aztfexport` suffix before the extension (e.g. `main.aztfexport.tf`), to avoid potential file name conflicts. If you run `aztfexport --append` multiple times, the generated config in `main.aztfexport.tf` will be appended in each run.
+    ```shell
+    aztfexport [subcommand] --backend-type=azurerm --backend-config=resource_group_name=<resource group name> --backend-config=storage_account_name=<account name> --backend-config=container_name=<container name> --backend-config=key=terraform.tfstate 
+    ```
+
+- Prepare the [`terraform` setting](https://developer.hashicorp.com/terraform/language/settings) in the output directory, and run `aztfexport [subcommand] --append`
 
 ### Config
 

command_before_func.go

@@ -0,0 +1,159 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/Azure/aztfexport/internal/meta"
+	"github.com/Azure/aztfexport/internal/utils"
+	"github.com/urfave/cli/v2"
+)
+
+func commandBeforeFunc(fset *FlagSet) func(ctx *cli.Context) error {
+	return func(_ *cli.Context) error {
+		// Common flags check
+		if fset.flagAppend {
+			if fset.flagOverwrite {
+				return fmt.Errorf("`--append` conflicts with `--overwrite`")
+			}
+		}
+		if !fset.flagNonInteractive {
+			if fset.flagContinue {
+				return fmt.Errorf("`--continue` must be used together with `--non-interactive`")
+			}
+			if fset.flagGenerateMappingFile {
+				return fmt.Errorf("`--generate-mapping-file` must be used together with `--non-interactive`")
+			}
+		}
+		if fset.flagHCLOnly {
+			if fset.flagAppend {
+				return fmt.Errorf("`--append` conflicts with `--hcl-only`")
+			}
+			if fset.flagModulePath != "" {
+				return fmt.Errorf("`--module-path` conflicts with `--hcl-only`")
+			}
+		}
+		if fset.flagModulePath != "" {
+			if !fset.flagAppend {
+				return fmt.Errorf("`--module-path` must be used together with `--append`")
+			}
+		}
+
+		if flagLogLevel != "" {
+			if _, err := logLevel(flagLogLevel); err != nil {
+				return err
+			}
+		}
+
+		// Initialize output directory
+		if _, err := os.Stat(fset.flagOutputDir); os.IsNotExist(err) {
+			if err := os.MkdirAll(fset.flagOutputDir, 0750); err != nil {
+				return fmt.Errorf("creating output directory %q: %v", fset.flagOutputDir, err)
+			}
+		}
+		empty, err := utils.DirIsEmpty(fset.flagOutputDir)
+		if err != nil {
+			return fmt.Errorf("failed to check emptiness of output directory %q: %v", fset.flagOutputDir, err)
+		}
+
+		var tfblock *utils.TerraformBlockDetail
+		if !empty {
+			switch {
+			case fset.flagOverwrite:
+				if err := utils.RemoveEverythingUnder(fset.flagOutputDir, meta.ResourceMappingFileName); err != nil {
+					return fmt.Errorf("failed to clean up output directory %q: %v", fset.flagOutputDir, err)
+				}
+			case fset.flagAppend:
+				tfblock, err = utils.InspecTerraformBlock(fset.flagOutputDir)
+				if err != nil {
+					return fmt.Errorf("determine the backend type from the existing files: %v", err)
+				}
+			default:
+				if fset.flagNonInteractive {
+					return fmt.Errorf("the output directory %q is not empty", fset.flagOutputDir)
+				}
+
+				// Interactive mode
+				fmt.Printf(`
+The output directory is not empty. Please choose one of actions below:
+
+* Press "Y" to overwrite the existing directory with new files
+* Press "N" to append new files and add to the existing state instead
+* Press other keys to quit
+
+> `)
+				var ans string
+				// #nosec G104
+				fmt.Scanf("%s", &ans)
+				switch strings.ToLower(ans) {
+				case "y":
+					if err := utils.RemoveEverythingUnder(fset.flagOutputDir, meta.ResourceMappingFileName); err != nil {
+						return err
+					}
+				case "n":
+					if fset.flagHCLOnly {
+						return fmt.Errorf("`--hcl-only` can only run within an empty directory. Use `-o` to specify an empty directory.")
+					}
+					fset.flagAppend = true
+					tfblock, err = utils.InspecTerraformBlock(fset.flagOutputDir)
+					if err != nil {
+						return fmt.Errorf("determine the backend type from the existing files: %v", err)
+					}
+				default:
+					return fmt.Errorf("the output directory %q is not empty", fset.flagOutputDir)
+				}
+			}
+		}
+
+		// Deterimine the real backend type to use
+		var existingBackendType string
+		if tfblock != nil {
+			existingBackendType = "local"
+			if tfblock.BackendType != "" {
+				existingBackendType = tfblock.BackendType
+			}
+		}
+		switch {
+		case fset.flagBackendType != "" && existingBackendType != "":
+			if fset.flagBackendType != existingBackendType {
+				return fmt.Errorf("the backend type defined in existing files (%s) are not the same as is specified in the CLI (%s)", existingBackendType, fset.flagBackendType)
+			}
+		case fset.flagBackendType == "" && existingBackendType == "":
+			fset.flagBackendType = "local"
+		case fset.flagBackendType == "" && existingBackendType != "":
+			fset.flagBackendType = existingBackendType
+		case fset.flagBackendType != "" && existingBackendType == "":
+			// do nothing
+		}
+
+		// Check backend related flags
+		if len(fset.flagBackendConfig.Value()) != 0 {
+			if existingBackendType != "" {
+				return fmt.Errorf("`--backend-config` should not be specified when appending to a workspace that has terraform block already defined")
+			}
+			if fset.flagBackendType == "local" {
+				return fmt.Errorf("`--backend-config` only works for non-local backend")
+			}
+		}
+		if fset.flagBackendType != "local" {
+			if fset.flagHCLOnly {
+				return fmt.Errorf("`--hcl-only` only works for local backend")
+			}
+		}
+
+		// Identify the subscription id, which comes from one of following (starts from the highest priority):
+		// - Command line option
+		// - Env variable: AZTFEXPORT_SUBSCRIPTION_ID
+		// - Env variable: ARM_SUBSCRIPTION_ID
+		// - Output of azure cli, the current active subscription
+		if fset.flagSubscriptionId == "" {
+			var err error
+			fset.flagSubscriptionId, err = subscriptionIdFromCLI()
+			if err != nil {
+				return fmt.Errorf("retrieving subscription id from CLI: %v", err)
+			}
+		}
+		return nil
+	}
+}

command_before_func_test.go

@@ -0,0 +1,232 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/urfave/cli/v2"
+)
+
+func TestCommondBeforeFunc(t *testing.T) {
+	dirGenEmpty := func(t *testing.T) string {
+		return t.TempDir()
+	}
+	dirGenWithTFBlock := func(content string) func(t *testing.T) string {
+		return func(t *testing.T) string {
+			dir := t.TempDir()
+			if err := os.WriteFile(filepath.Join(dir, "terraform.tf"), []byte(content), 0640); err != nil {
+				t.Fatal(err)
+			}
+			return dir
+		}
+	}
+
+	cases := []struct {
+		name      string
+		fset      FlagSet
+		dirGen    func(t *testing.T) string
+		err       string
+		postCheck func(t *testing.T, flagset FlagSet)
+	}{
+		{
+			name: "--append conflicts with --overwrite",
+			fset: FlagSet{
+				flagAppend:    true,
+				flagOverwrite: true,
+			},
+			err: "`--append` conflicts with `--overwrite`",
+		},
+		{
+			name: "only a --append works",
+			fset: FlagSet{
+				flagAppend:         true,
+				flagSubscriptionId: "123",
+			},
+		},
+		{
+			name: "only a --overwrite works",
+			fset: FlagSet{
+				flagOverwrite: true,
+			},
+		},
+		{
+			name: "--continue shouldn't be used in interactive mode since interactive mode can toggle off the failed resources",
+			fset: FlagSet{
+				flagContinue: true,
+			},
+			err: "`--continue` must be used together with `--non-interactive`",
+		},
+		{
+			name: "--continue with --non-interactive works",
+			fset: FlagSet{
+				flagContinue:       true,
+				flagNonInteractive: true,
+			},
+		},
+		{
+			name: "--generate-mapping-file shouldn't be used in interactive mode since interactive mode has a special code to do it",
+			fset: FlagSet{
+				flagGenerateMappingFile: true,
+			},
+			err: "`--generate-mapping-file` must be used together with `--non-interactive`",
+		},
+		{
+			name: "--generate-mapping-file with --non-interactive works",
+			fset: FlagSet{
+				flagGenerateMappingFile: true,
+				flagNonInteractive:      true,
+			},
+		},
+		{
+			name: "--hcl-only shouldn't be used with --append since it doesn't make sense to generate config/state to an existing workspace for hcl only",
+			fset: FlagSet{
+				flagHCLOnly: true,
+				flagAppend:  true,
+			},
+			err: "`--append` conflicts with `--hcl-only`",
+		},
+		{
+			name: "--hcl-only works alone",
+			fset: FlagSet{
+				flagHCLOnly: true,
+			},
+		},
+		{
+			name: "--module-path shouldn't be used with --hcl-only since --module-path will be used together with --append",
+			fset: FlagSet{
+				flagHCLOnly:    true,
+				flagModulePath: "foo",
+			},
+			err: "`--module-path` conflicts with `--hcl-only`",
+		},
+		{
+			name: "--module-path should be used together with --append",
+			fset: FlagSet{
+				flagModulePath: "foo",
+			},
+			err: "`--module-path` must be used together with `--append`",
+		},
+		{
+			name: "--module-path with --append works",
+			fset: FlagSet{
+				flagModulePath: "foo",
+				flagAppend:     true,
+			},
+		},
+		{
+			name: "non empty dir but overwrite",
+			fset: FlagSet{
+				flagOverwrite: true,
+			},
+			dirGen: dirGenWithTFBlock("foo {}"),
+		},
+		{
+			name: "default backend type is local",
+			fset: FlagSet{},
+			postCheck: func(t *testing.T, flagset FlagSet) {
+				require.Equal(t, "local", flagset.flagBackendType)
+			},
+		},
+		{
+			name: "append to a dir with no terraform config ends up backend type local",
+			fset: FlagSet{
+				flagAppend: true,
+			},
+			dirGen: dirGenWithTFBlock("foo {}"),
+			postCheck: func(t *testing.T, flagset FlagSet) {
+				require.Equal(t, "local", flagset.flagBackendType)
+			},
+		},
+		{
+			name: "append to a dir with empty terraform config ends up backend type local, which conflicts with the specified backend type",
+			fset: FlagSet{
+				flagBackendType: "azurerm",
+				flagAppend:      true,
+			},
+			dirGen: dirGenWithTFBlock(`terraform {}`),
+			err:    "the backend type defined in existing files (local) are not the same as is specified in the CLI (azurerm)",
+		},
+		{
+			name: "append to a dir with terraform config of backend type set to local, which conflicts with the specified backend type",
+			fset: FlagSet{
+				flagBackendType: "azurerm",
+				flagAppend:      true,
+			},
+			dirGen: dirGenWithTFBlock(`terraform {
+	backend local {}
+}`),
+			err: "the backend type defined in existing files (local) are not the same as is specified in the CLI (azurerm)",
+		},
+		{
+			name: "append to a dir with terraform config of backend type set to azurerm, which aligns with the specified backend type",
+			fset: FlagSet{
+				flagBackendType: "azurerm",
+				flagAppend:      true,
+			},
+			dirGen: dirGenWithTFBlock(`terraform {
+	backend azurerm {}
+}`),
+		},
+		{
+			name: "append to a dir with terraform config of backend type set to foo, which conflicts with the specified backend type",
+			fset: FlagSet{
+				flagBackendType: "azurerm",
+				flagAppend:      true,
+			},
+			dirGen: dirGenWithTFBlock(`terraform {
+	backend foo {}
+}`),
+			err: "the backend type defined in existing files (foo) are not the same as is specified in the CLI (azurerm)",
+		},
+		{
+			name: "--backend-config shouldn't be used with local backend",
+			fset: FlagSet{
+				flagBackendConfig: *cli.NewStringSlice("foo=bar"),
+			},
+			err: "`--backend-config` only works for non-local backend",
+		},
+		{
+			name: "--backend-config shouldn't be used when appending to a workspace with backend config defined",
+			fset: FlagSet{
+				flagAppend:        true,
+				flagBackendConfig: *cli.NewStringSlice("foo=bar"),
+			},
+			dirGen: dirGenWithTFBlock(`terraform {}`),
+			err:    "`--backend-config` should not be specified when appending to a workspace that has terraform block already defined",
+		},
+		{
+			name: "--hcl-only can't work for remote backend",
+			fset: FlagSet{
+				flagBackendType: "azurerm",
+				flagHCLOnly:     true,
+			},
+			err: "`--hcl-only` only works for local backend",
+		},
+	}
+
+	for _, tt := range cases {
+		t.Run(tt.name, func(t *testing.T) {
+			if tt.dirGen == nil {
+				tt.dirGen = dirGenEmpty
+			}
+			tt.fset.flagOutputDir = tt.dirGen(t)
+
+			// This is to avoid reading the subscription id from az cli, which is not setup in CI.
+			if tt.fset.flagSubscriptionId == "" {
+				tt.fset.flagSubscriptionId = "test"
+			}
+
+			err := commandBeforeFunc(&tt.fset)(nil)
+			if tt.err == "" {
+				require.NoError(t, err)
+				if tt.postCheck != nil {
+					tt.postCheck(t, tt.fset)
+				}
+				return
+			}
+			require.ErrorContains(t, err, tt.err)
+		})
+	}
+}

flag.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"github.com/urfave/cli/v2"
+)
+
+var flagset FlagSet
+
+type FlagSet struct {
+	// common flags
+	flagSubscriptionId      string
+	flagOutputDir           string
+	flagOverwrite           bool
+	flagAppend              bool
+	flagDevProvider         bool
+	flagBackendType         string
+	flagBackendConfig       cli.StringSlice
+	flagFullConfig          bool
+	flagParallelism         int
+	flagContinue            bool
+	flagNonInteractive      bool
+	flagGenerateMappingFile bool
+	flagHCLOnly             bool
+	flagModulePath          string
+
+	// common flags (hidden)
+	hflagMockClient bool
+	hflagPlainUI    bool
+	hflagProfile    string
+
+	// Subcommand specific flags
+	//
+	// res:
+	// flagResName
+	// flagResType
+	//
+	// rg:
+	// flagPattern
+	//
+	// query:
+	// flagPattern
+	// flagRecursive
+	flagPattern   string
+	flagRecursive bool
+	flagResName   string
+	flagResType   string
+}

internal/meta/base_meta.go

@@ -637,7 +637,11 @@ func (meta *baseMeta) initTF(ctx context.Context) error {
 func (meta *baseMeta) initProvider(ctx context.Context) error {
 	log.Printf("[INFO] Init provider")
 
-	module, err := tfconfig.LoadModule(meta.outdir)
+	module, diags := tfconfig.LoadModule(meta.outdir)
+	if diags.HasErrors() {
+		return diags.Err()
+	}
+	tfblock, err := utils.InspecTerraformBlock(meta.outdir)
 	if err != nil {
 		return err
 	}
@@ -651,8 +655,8 @@ func (meta *baseMeta) initProvider(ctx context.Context) error {
 		}
 	}
 
-	if len(module.ProviderConfigs) == 0 {
-		log.Printf("[INFO] Output directory doesn't contain terraform required provider setting, create one then")
+	if tfblock == nil {
+		log.Printf("[INFO] Output directory doesn't contain terraform block, create one then")
 		cfgFile := filepath.Join(meta.outdir, meta.outputFileNames.TerraformFileName)
 		// #nosec G306
 		if err := os.WriteFile(cfgFile, []byte(meta.buildTerraformConfig(meta.backendType)), 0644); err != nil {

internal/utils/inspect_terraform.go

@@ -0,0 +1,56 @@
+package utils
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/hashicorp/hcl/v2"
+	"github.com/hashicorp/hcl/v2/hclsyntax"
+)
+
+type TerraformBlockDetail struct {
+	BackendType string
+}
+
+// InspecTerraformBlock inspects the terraform block by interating the top level .tf files.
+// This function assumes the dir is a valid terraform workspace, which means there is at most one terraform block defined.
+func InspecTerraformBlock(dir string) (*TerraformBlockDetail, error) {
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return nil, fmt.Errorf("reading directory %s: %v", dir, err)
+	}
+
+	for _, entry := range entries {
+		if entry.IsDir() {
+			continue
+		}
+		if filepath.Ext(entry.Name()) != ".tf" {
+			continue
+		}
+
+		// #nosec G304
+		b, err := os.ReadFile(filepath.Join(dir, entry.Name()))
+		if err != nil {
+			return nil, fmt.Errorf("reading file %s: %v", entry.Name(), err)
+		}
+		f, diags := hclsyntax.ParseConfig(b, entry.Name(), hcl.InitialPos)
+		if diags.HasErrors() {
+			return nil, fmt.Errorf("parsing file %s: %v", entry.Name(), diags.Error())
+		}
+		for _, block := range f.Body.(*hclsyntax.Body).Blocks {
+			if block.Type != "terraform" {
+				continue
+			}
+			var detail TerraformBlockDetail
+			for _, block := range block.Body.Blocks {
+				switch block.Type {
+				case "backend":
+					detail.BackendType = block.Labels[0]
+				}
+			}
+			return &detail, nil
+		}
+	}
+	return nil, nil
+}

main.go

@@ -16,7 +16,6 @@ import (
 
 	"github.com/Azure/aztfexport/internal/cfgfile"
 	internalconfig "github.com/Azure/aztfexport/internal/config"
-	"github.com/Azure/aztfexport/internal/meta"
 	"github.com/Azure/aztfexport/pkg/telemetry"
 	"github.com/gofrs/uuid"
 	"github.com/pkg/profile"
@@ -31,7 +30,6 @@ import (
 
 	"github.com/Azure/aztfexport/internal"
 	"github.com/Azure/aztfexport/internal/ui"
-	"github.com/Azure/aztfexport/internal/utils"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/arm"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud"
@@ -46,218 +44,70 @@ var (
 	flagLogLevel string
 )
 
+func prepareConfigFile(ctx *cli.Context) error {
+	// Prepare the config directory at $HOME/.aztfexport
+	homeDir, err := os.UserHomeDir()
+	if err != nil {
+		return fmt.Errorf("retrieving the user's HOME directory: %v", err)
+	}
+	configDir := filepath.Join(homeDir, cfgfile.CfgDirName)
+	if err := os.MkdirAll(configDir, 0750); err != nil {
+		return fmt.Errorf("creating the config directory at %s: %v", configDir, err)
+	}
+	configFile := filepath.Join(configDir, cfgfile.CfgFileName)
+
+	_, err = os.Stat(configFile)
+	if err == nil {
+		return nil
+	}
+	if !os.IsNotExist(err) {
+		return nil
+	}
+
+	// Generate a configuration file if not exist.
+
+	// Get the installation id from following sources in order:
+	// 1. The Azure CLI's configuration file
+	// 2. The Azure PWSH's configuration file
+	// 3. Generate one
+	id, err := func() (string, error) {
+		if id, err := cfgfile.GetInstallationIdFromCLI(); err == nil {
+			return id, nil
+		}
+		log.Printf("[DEBUG] Installation ID not found from Azure CLI: %v", err)
+
+		if id, err := cfgfile.GetInstallationIdFromPWSH(); err == nil {
+			return id, nil
+		}
+		log.Printf("[DEBUG] Installation ID not found from Azure PWSH: %v", err)
+
+		uuid, err := uuid.NewV4()
+		if err != nil {
+			return "", fmt.Errorf("generating installation id: %w", err)
+		}
+		return uuid.String(), nil
+	}()
+
+	if err != nil {
+		return err
+	}
+
+	cfg := cfgfile.Configuration{
+		InstallationId:   id,
+		TelemetryEnabled: true,
+	}
+	b, err := json.Marshal(cfg)
+	if err != nil {
+		return fmt.Errorf("marshalling the configuration file: %v", err)
+	}
+	// #nosec G306
+	if err := os.WriteFile(configFile, b, 0644); err != nil {
+		return fmt.Errorf("writing the configuration file: %v", err)
+	}
+	return nil
+}
+
 func main() {
-	var (
-		// common flags
-		flagSubscriptionId      string
-		flagOutputDir           string
-		flagOverwrite           bool
-		flagAppend              bool
-		flagDevProvider         bool
-		flagBackendType         string
-		flagBackendConfig       cli.StringSlice
-		flagFullConfig          bool
-		flagParallelism         int
-		flagContinue            bool
-		flagNonInteractive      bool
-		flagGenerateMappingFile bool
-		flagHCLOnly             bool
-		flagModulePath          string
-
-		// common flags (hidden)
-		hflagMockClient bool
-		hflagPlainUI    bool
-		hflagProfile    string
-
-		// Subcommand specific flags
-		//
-		// res:
-		// flagResName
-		// flagResType
-		//
-		// rg:
-		// flagPattern
-		//
-		// query:
-		// flagPattern
-		// flagRecursive
-		flagPattern   string
-		flagRecursive bool
-		flagResName   string
-		flagResType   string
-	)
-
-	prepareConfigFile := func(ctx *cli.Context) error {
-		// Prepare the config directory at $HOME/.aztfexport
-		homeDir, err := os.UserHomeDir()
-		if err != nil {
-			return fmt.Errorf("retrieving the user's HOME directory: %v", err)
-		}
-		configDir := filepath.Join(homeDir, cfgfile.CfgDirName)
-		if err := os.MkdirAll(configDir, 0750); err != nil {
-			return fmt.Errorf("creating the config directory at %s: %v", configDir, err)
-		}
-		configFile := filepath.Join(configDir, cfgfile.CfgFileName)
-
-		_, err = os.Stat(configFile)
-		if err == nil {
-			return nil
-		}
-		if !os.IsNotExist(err) {
-			return nil
-		}
-
-		// Generate a configuration file if not exist.
-
-		// Get the installation id from following sources in order:
-		// 1. The Azure CLI's configuration file
-		// 2. The Azure PWSH's configuration file
-		// 3. Generate one
-		id, err := func() (string, error) {
-			if id, err := cfgfile.GetInstallationIdFromCLI(); err == nil {
-				return id, nil
-			}
-			log.Printf("[DEBUG] Installation ID not found from Azure CLI: %v", err)
-
-			if id, err := cfgfile.GetInstallationIdFromPWSH(); err == nil {
-				return id, nil
-			}
-			log.Printf("[DEBUG] Installation ID not found from Azure PWSH: %v", err)
-
-			uuid, err := uuid.NewV4()
-			if err != nil {
-				return "", fmt.Errorf("generating installation id: %w", err)
-			}
-			return uuid.String(), nil
-		}()
-
-		if err != nil {
-			return err
-		}
-
-		cfg := cfgfile.Configuration{
-			InstallationId:   id,
-			TelemetryEnabled: true,
-		}
-		b, err := json.Marshal(cfg)
-		if err != nil {
-			return fmt.Errorf("marshalling the configuration file: %v", err)
-		}
-		// #nosec G306
-		if err := os.WriteFile(configFile, b, 0644); err != nil {
-			return fmt.Errorf("writing the configuration file: %v", err)
-		}
-		return nil
-	}
-
-	commandBeforeFunc := func(ctx *cli.Context) error {
-		// Common flags check
-		if flagAppend {
-			if flagBackendType != "local" {
-				return fmt.Errorf("`--append` only works for local backend")
-			}
-			if flagOverwrite {
-				return fmt.Errorf("`--append` conflicts with `--overwrite`")
-			}
-		}
-		if !flagNonInteractive {
-			if flagContinue {
-				return fmt.Errorf("`--continue` must be used together with `--non-interactive`")
-			}
-			if flagGenerateMappingFile {
-				return fmt.Errorf("`--generate-mapping-file` must be used together with `--non-interactive`")
-			}
-		}
-		if flagHCLOnly {
-			if flagBackendType != "local" {
-				return fmt.Errorf("`--hcl-only` only works for local backend")
-			}
-			if flagAppend {
-				return fmt.Errorf("`--append` conflicts with `--hcl-only`")
-			}
-			if flagModulePath != "" {
-				return fmt.Errorf("`--module-path` conflicts with `--hcl-only`")
-			}
-		}
-		if flagModulePath != "" {
-			if !flagAppend {
-				return fmt.Errorf("`--module-path` must be used together with `--append`")
-			}
-		}
-
-		if flagLogLevel != "" {
-			if _, err := logLevel(flagLogLevel); err != nil {
-				return err
-			}
-		}
-
-		// Initialize output directory
-		if _, err := os.Stat(flagOutputDir); os.IsNotExist(err) {
-			if err := os.MkdirAll(flagOutputDir, 0750); err != nil {
-				return fmt.Errorf("creating output directory %q: %v", flagOutputDir, err)
-			}
-		}
-		empty, err := utils.DirIsEmpty(flagOutputDir)
-		if err != nil {
-			return fmt.Errorf("failed to check emptiness of output directory %q: %v", flagOutputDir, err)
-		}
-		if !empty {
-			switch {
-			case flagOverwrite:
-				if err := utils.RemoveEverythingUnder(flagOutputDir, meta.ResourceMappingFileName); err != nil {
-					return fmt.Errorf("failed to clean up output directory %q: %v", flagOutputDir, err)
-				}
-			case flagAppend:
-				// do nothing
-			default:
-				if flagNonInteractive {
-					return fmt.Errorf("the output directory %q is not empty", flagOutputDir)
-				}
-
-				// Interactive mode
-				fmt.Printf(`
-The output directory is not empty. Please choose one of actions below:
-
-* Press "Y" to overwrite the existing directory with new files
-* Press "N" to append new files and add to the existing state instead
-* Press other keys to quit
-
-> `)
-				var ans string
-				// #nosec G104
-				fmt.Scanf("%s", &ans)
-				switch strings.ToLower(ans) {
-				case "y":
-					if err := utils.RemoveEverythingUnder(flagOutputDir, meta.ResourceMappingFileName); err != nil {
-						return err
-					}
-				case "n":
-					if flagHCLOnly {
-						return fmt.Errorf("`--hcl-only` can only run within an empty directory. Use `-o` to specify an empty directory.")
-					}
-					flagAppend = true
-				default:
-					return fmt.Errorf("the output directory %q is not empty", flagOutputDir)
-				}
-			}
-		}
-
-		// Identify the subscription id, which comes from one of following (starts from the highest priority):
-		// - Command line option
-		// - Env variable: AZTFEXPORT_SUBSCRIPTION_ID
-		// - Env variable: ARM_SUBSCRIPTION_ID
-		// - Output of azure cli, the current active subscription
-		if flagSubscriptionId == "" {
-			var err error
-			flagSubscriptionId, err = subscriptionIdFromCLI()
-			if err != nil {
-				return fmt.Errorf("retrieving subscription id from CLI: %v", err)
-			}
-		}
-
-		return nil
-	}
-
 	commonFlags := []cli.Flag{
 		&cli.StringFlag{
 			Name: "subscription-id",
@@ -265,7 +115,7 @@ The output directory is not empty. Please choose one of actions below:
 			EnvVars:     []string{"AZTFEXPORT_SUBSCRIPTION_ID", "ARM_SUBSCRIPTION_ID"},
 			Aliases:     []string{"s"},
 			Usage:       "The subscription id",
-			Destination: &flagSubscriptionId,
+			Destination: &flagset.flagSubscriptionId,
 		},
 		&cli.StringFlag{
 			Name:    "output-dir",
@@ -276,86 +126,85 @@ The output directory is not empty. Please choose one of actions below:
 				dir, _ := os.Getwd()
 				return dir
 			}(),
-			Destination: &flagOutputDir,
+			Destination: &flagset.flagOutputDir,
 		},
 		&cli.BoolFlag{
 			Name:        "overwrite",
 			EnvVars:     []string{"AZTFEXPORT_OVERWRITE"},
 			Aliases:     []string{"f"},
 			Usage:       "Overwrites the output directory if it is not empty (use with caution)",
-			Destination: &flagOverwrite,
+			Destination: &flagset.flagOverwrite,
 		},
 		&cli.BoolFlag{
 			Name:        "append",
 			EnvVars:     []string{"AZTFEXPORT_APPEND"},
-			Usage:       "Imports to the existing state file if any and does not clean up the output directory (local backend only)",
-			Destination: &flagAppend,
+			Usage:       "Imports to the existing state file if any and does not clean up the output directory",
+			Destination: &flagset.flagAppend,
 		},
 		&cli.BoolFlag{
 			Name:        "dev-provider",
 			EnvVars:     []string{"AZTFEXPORT_DEV_PROVIDER"},
 			Usage:       fmt.Sprintf("Use the local development AzureRM provider, instead of the pinned provider in v%s", azurerm.ProviderSchemaInfo.Version),
-			Destination: &flagDevProvider,
+			Destination: &flagset.flagDevProvider,
 		},
 		&cli.StringFlag{
 			Name:        "backend-type",
 			EnvVars:     []string{"AZTFEXPORT_BACKEND_TYPE"},
-			Usage:       "The Terraform backend used to store the state",
-			Value:       "local",
-			Destination: &flagBackendType,
+			Usage:       "The Terraform backend used to store the state (default: local)",
+			Destination: &flagset.flagBackendType,
 		},
 		&cli.StringSliceFlag{
 			Name:        "backend-config",
 			EnvVars:     []string{"AZTFEXPORT_BACKEND_CONFIG"},
 			Usage:       "The Terraform backend config",
-			Destination: &flagBackendConfig,
+			Destination: &flagset.flagBackendConfig,
 		},
 		&cli.BoolFlag{
 			Name:        "full-properties",
 			EnvVars:     []string{"AZTFEXPORT_FULL_PROPERTIES"},
 			Usage:       "Includes all non-computed properties in the Terraform configuration. This may require manual modifications to produce a valid config",
 			Value:       false,
-			Destination: &flagFullConfig,
+			Destination: &flagset.flagFullConfig,
 		},
 		&cli.IntFlag{
 			Name:        "parallelism",
 			EnvVars:     []string{"AZTFEXPORT_PARALLELISM"},
 			Usage:       "Limit the number of parallel operations, i.e., resource discovery, import",
 			Value:       10,
-			Destination: &flagParallelism,
+			Destination: &flagset.flagParallelism,
 		},
 		&cli.BoolFlag{
 			Name:        "non-interactive",
 			EnvVars:     []string{"AZTFEXPORT_NON_INTERACTIVE"},
 			Aliases:     []string{"n"},
 			Usage:       "Non-interactive mode",
-			Destination: &flagNonInteractive,
+			Destination: &flagset.flagNonInteractive,
 		},
 		&cli.BoolFlag{
 			Name:        "continue",
 			EnvVars:     []string{"AZTFEXPORT_CONTINUE"},
 			Aliases:     []string{"k"},
 			Usage:       "For non-interactive mode, continue on any import error",
-			Destination: &flagContinue,
+			Destination: &flagset.flagContinue,
 		},
 		&cli.BoolFlag{
 			Name:        "generate-mapping-file",
 			Aliases:     []string{"g"},
 			EnvVars:     []string{"AZTFEXPORT_GENERATE_MAPPING_FILE"},
 			Usage:       "Only generate the resource mapping file, but does NOT import any resource",
-			Destination: &flagGenerateMappingFile,
+			Destination: &flagset.flagGenerateMappingFile,
 		},
 		&cli.BoolFlag{
 			Name:        "hcl-only",
 			EnvVars:     []string{"AZTFEXPORT_HCL_ONLY"},
 			Usage:       "Only generates HCL code (and mapping file), but not the files for resource management (e.g. the state file)",
-			Destination: &flagHCLOnly,
+			Destination: &flagset.flagHCLOnly,
 		},
 		&cli.StringFlag{
 			Name:        "module-path",
 			EnvVars:     []string{"AZTFEXPORT_MODULE_PATH"},
 			Usage:       `The path of the module (e.g. "module1.module2") where the resources will be imported and config generated. Note that only modules whose "source" is local path is supported. Defaults to the root module.`,
-			Destination: &flagModulePath,
+			Destination: &flagset.flagModulePath,
 		},
 		&cli.StringFlag{
 			Name:        "log-path",
@@ -377,21 +226,21 @@ The output directory is not empty. Please choose one of actions below:
 			EnvVars:     []string{"AZTFEXPORT_MOCK_CLIENT"},
 			Usage:       "Whether to mock the client. This is for testing UI",
 			Hidden:      true,
-			Destination: &hflagMockClient,
+			Destination: &flagset.hflagMockClient,
 		},
 		&cli.BoolFlag{
 			Name:        "plain-ui",
 			EnvVars:     []string{"AZTFEXPORT_PLAIN_UI"},
 			Usage:       "In non-interactive mode, print the progress information line by line, rather than the spinner UI",
 			Hidden:      true,
-			Destination: &hflagPlainUI,
+			Destination: &flagset.hflagPlainUI,
 		},
 		&cli.StringFlag{
 			Name:        "profile",
 			EnvVars:     []string{"AZTFEXPORT_PROFILE"},
 			Usage:       "Profile the program, possible values are `cpu` and `memory`",
 			Hidden:      true,
-			Destination: &hflagProfile,
+			Destination: &flagset.hflagProfile,
 		},
 	}
 
@@ -401,13 +250,13 @@ The output directory is not empty. Please choose one of actions below:
 			EnvVars:     []string{"AZTFEXPORT_NAME"},
 			Usage:       `The Terraform resource name.`,
 			Value:       "res-0",
-			Destination: &flagResName,
+			Destination: &flagset.flagResName,
 		},
 		&cli.StringFlag{
 			Name:        "type",
 			EnvVars:     []string{"AZTFEXPORT_TYPE"},
 			Usage:       `The Terraform resource type.`,
-			Destination: &flagResType,
+			Destination: &flagset.flagResType,
 		},
 	}, commonFlags...)
 
@@ -418,7 +267,7 @@ The output directory is not empty. Please choose one of actions below:
 			Aliases:     []string{"p"},
 			Usage:       `The pattern of the resource name. The semantic of a pattern is the same as Go's os.CreateTemp()`,
 			Value:       "res-",
-			Destination: &flagPattern,
+			Destination: &flagset.flagPattern,
 		},
 	}, commonFlags...)
 
@@ -428,7 +277,7 @@ The output directory is not empty. Please choose one of actions below:
 			EnvVars:     []string{"AZTFEXPORT_RECURSIVE"},
 			Aliases:     []string{"r"},
 			Usage:       "Recursively lists child resources of the resulting query resources",
-			Destination: &flagRecursive,
+			Destination: &flagset.flagRecursive,
 		},
 	}, resourceGroupFlags...)
 
@@ -510,7 +359,7 @@ The output directory is not empty. Please choose one of actions below:
 				Usage:     "Exporting a single resource",
 				UsageText: "aztfexport resource [option] <resource id>",
 				Flags:     resourceFlags,
-				Before:    commandBeforeFunc,
+				Before:    commandBeforeFunc(&flagset),
 				Action: func(c *cli.Context) error {
 					if c.NArg() == 0 {
 						return fmt.Errorf("No resource id specified")
@@ -533,30 +382,30 @@ The output directory is not empty. Please choose one of actions below:
 					// Initialize the config
 					cfg := config.Config{
 						CommonConfig: config.CommonConfig{
-							SubscriptionId:       flagSubscriptionId,
+							SubscriptionId:       flagset.flagSubscriptionId,
 							AzureSDKCredential:   cred,
 							AzureSDKClientOption: *clientOpt,
-							OutputDir:            flagOutputDir,
-							DevProvider:          flagDevProvider,
-							ContinueOnError:      flagContinue,
-							BackendType:          flagBackendType,
-							BackendConfig:        flagBackendConfig.Value(),
-							FullConfig:           flagFullConfig,
-							Parallelism:          flagParallelism,
-							HCLOnly:              flagHCLOnly,
-							ModulePath:           flagModulePath,
+							OutputDir:            flagset.flagOutputDir,
+							DevProvider:          flagset.flagDevProvider,
+							ContinueOnError:      flagset.flagContinue,
+							BackendType:          flagset.flagBackendType,
+							BackendConfig:        flagset.flagBackendConfig.Value(),
+							FullConfig:           flagset.flagFullConfig,
+							Parallelism:          flagset.flagParallelism,
+							HCLOnly:              flagset.flagHCLOnly,
+							ModulePath:           flagset.flagModulePath,
 							TelemetryClient:      initTelemetryClient(),
 						},
 						ResourceId:     resId,
-						TFResourceName: flagResName,
-						TFResourceType: flagResType,
+						TFResourceName: flagset.flagResName,
+						TFResourceType: flagset.flagResType,
 					}
 
-					if flagAppend {
+					if flagset.flagAppend {
 						cfg.CommonConfig.OutputFileNames = safeOutputFileNames
 					}
 
-					return realMain(c.Context, cfg, flagNonInteractive, hflagMockClient, hflagPlainUI, flagGenerateMappingFile, hflagProfile)
+					return realMain(c.Context, cfg, flagset.flagNonInteractive, flagset.hflagMockClient, flagset.hflagPlainUI, flagset.flagGenerateMappingFile, flagset.hflagProfile)
 				},
 			},
 			{
@@ -565,7 +414,7 @@ The output directory is not empty. Please choose one of actions below:
 				Usage:     "Exporting a resource group and the nested resources resides within it",
 				UsageText: "aztfexport resource-group [option] <resource group name>",
 				Flags:     resourceGroupFlags,
-				Before:    commandBeforeFunc,
+				Before:    commandBeforeFunc(&flagset),
 				Action: func(c *cli.Context) error {
 					if c.NArg() == 0 {
 						return fmt.Errorf("No resource group specified")
@@ -584,30 +433,30 @@ The output directory is not empty. Please choose one of actions below:
 					// Initialize the config
 					cfg := config.Config{
 						CommonConfig: config.CommonConfig{
-							SubscriptionId:       flagSubscriptionId,
+							SubscriptionId:       flagset.flagSubscriptionId,
 							AzureSDKCredential:   cred,
 							AzureSDKClientOption: *clientOpt,
-							OutputDir:            flagOutputDir,
-							DevProvider:          flagDevProvider,
-							ContinueOnError:      flagContinue,
-							BackendType:          flagBackendType,
-							BackendConfig:        flagBackendConfig.Value(),
-							FullConfig:           flagFullConfig,
-							Parallelism:          flagParallelism,
-							HCLOnly:              flagHCLOnly,
-							ModulePath:           flagModulePath,
+							OutputDir:            flagset.flagOutputDir,
+							DevProvider:          flagset.flagDevProvider,
+							ContinueOnError:      flagset.flagContinue,
+							BackendType:          flagset.flagBackendType,
+							BackendConfig:        flagset.flagBackendConfig.Value(),
+							FullConfig:           flagset.flagFullConfig,
+							Parallelism:          flagset.flagParallelism,
+							HCLOnly:              flagset.flagHCLOnly,
+							ModulePath:           flagset.flagModulePath,
 							TelemetryClient:      initTelemetryClient(),
 						},
 						ResourceGroupName:   rg,
-						ResourceNamePattern: flagPattern,
+						ResourceNamePattern: flagset.flagPattern,
 						RecursiveQuery:      true,
 					}
 
-					if flagAppend {
+					if flagset.flagAppend {
 						cfg.CommonConfig.OutputFileNames = safeOutputFileNames
 					}
 
-					return realMain(c.Context, cfg, flagNonInteractive, hflagMockClient, hflagPlainUI, flagGenerateMappingFile, hflagProfile)
+					return realMain(c.Context, cfg, flagset.flagNonInteractive, flagset.hflagMockClient, flagset.hflagPlainUI, flagset.flagGenerateMappingFile, flagset.hflagProfile)
 				},
 			},
 			{
@@ -615,7 +464,7 @@ The output directory is not empty. Please choose one of actions below:
 				Usage:     "Exporting a customized scope of resources determined by an Azure Resource Graph where predicate",
 				UsageText: "aztfexport query [option] <ARG where predicate>",
 				Flags:     queryFlags,
-				Before:    commandBeforeFunc,
+				Before:    commandBeforeFunc(&flagset),
 				Action: func(c *cli.Context) error {
 					if c.NArg() == 0 {
 						return fmt.Errorf("No query specified")
@@ -634,30 +483,30 @@ The output directory is not empty. Please choose one of actions below:
 					// Initialize the config
 					cfg := config.Config{
 						CommonConfig: config.CommonConfig{
-							SubscriptionId:       flagSubscriptionId,
+							SubscriptionId:       flagset.flagSubscriptionId,
 							AzureSDKCredential:   cred,
 							AzureSDKClientOption: *clientOpt,
-							OutputDir:            flagOutputDir,
-							DevProvider:          flagDevProvider,
-							ContinueOnError:      flagContinue,
-							BackendType:          flagBackendType,
-							BackendConfig:        flagBackendConfig.Value(),
-							FullConfig:           flagFullConfig,
-							Parallelism:          flagParallelism,
-							HCLOnly:              flagHCLOnly,
-							ModulePath:           flagModulePath,
+							OutputDir:            flagset.flagOutputDir,
+							DevProvider:          flagset.flagDevProvider,
+							ContinueOnError:      flagset.flagContinue,
+							BackendType:          flagset.flagBackendType,
+							BackendConfig:        flagset.flagBackendConfig.Value(),
+							FullConfig:           flagset.flagFullConfig,
+							Parallelism:          flagset.flagParallelism,
+							HCLOnly:              flagset.flagHCLOnly,
+							ModulePath:           flagset.flagModulePath,
 							TelemetryClient:      initTelemetryClient(),
 						},
 						ARGPredicate:        predicate,
-						ResourceNamePattern: flagPattern,
-						RecursiveQuery:      flagRecursive,
+						ResourceNamePattern: flagset.flagPattern,
+						RecursiveQuery:      flagset.flagRecursive,
 					}
 
-					if flagAppend {
+					if flagset.flagAppend {
 						cfg.CommonConfig.OutputFileNames = safeOutputFileNames
 					}
 
-					return realMain(c.Context, cfg, flagNonInteractive, hflagMockClient, hflagPlainUI, flagGenerateMappingFile, hflagProfile)
+					return realMain(c.Context, cfg, flagset.flagNonInteractive, flagset.hflagMockClient, flagset.hflagPlainUI, flagset.flagGenerateMappingFile, flagset.hflagProfile)
 				},
 			},
 			{
@@ -666,7 +515,7 @@ The output directory is not empty. Please choose one of actions below:
 				Usage:     "Exporting a customized scope of resources determined by the resource mapping file",
 				UsageText: "aztfexport mapping-file [option] <resource mapping file>",
 				Flags:     mappingFileFlags,
-				Before:    commandBeforeFunc,
+				Before:    commandBeforeFunc(&flagset),
 				Action: func(c *cli.Context) error {
 					if c.NArg() == 0 {
 						return fmt.Errorf("No resource mapping file specified")
@@ -685,28 +534,28 @@ The output directory is not empty. Please choose one of actions below:
 					// Initialize the config
 					cfg := config.Config{
 						CommonConfig: config.CommonConfig{
-							SubscriptionId:       flagSubscriptionId,
+							SubscriptionId:       flagset.flagSubscriptionId,
 							AzureSDKCredential:   cred,
 							AzureSDKClientOption: *clientOpt,
-							OutputDir:            flagOutputDir,
-							DevProvider:          flagDevProvider,
-							ContinueOnError:      flagContinue,
-							BackendType:          flagBackendType,
-							BackendConfig:        flagBackendConfig.Value(),
-							FullConfig:           flagFullConfig,
-							Parallelism:          flagParallelism,
-							HCLOnly:              flagHCLOnly,
-							ModulePath:           flagModulePath,
+							OutputDir:            flagset.flagOutputDir,
+							DevProvider:          flagset.flagDevProvider,
+							ContinueOnError:      flagset.flagContinue,
+							BackendType:          flagset.flagBackendType,
+							BackendConfig:        flagset.flagBackendConfig.Value(),
+							FullConfig:           flagset.flagFullConfig,
+							Parallelism:          flagset.flagParallelism,
+							HCLOnly:              flagset.flagHCLOnly,
+							ModulePath:           flagset.flagModulePath,
 							TelemetryClient:      initTelemetryClient(),
 						},
 						MappingFile: mapFile,
 					}
 
-					if flagAppend {
+					if flagset.flagAppend {
 						cfg.CommonConfig.OutputFileNames = safeOutputFileNames
 					}
 
-					return realMain(c.Context, cfg, flagNonInteractive, hflagMockClient, hflagPlainUI, flagGenerateMappingFile, hflagProfile)
+					return realMain(c.Context, cfg, flagset.flagNonInteractive, flagset.hflagMockClient, flagset.hflagPlainUI, flagset.flagGenerateMappingFile, flagset.hflagProfile)
 				},
 			},
 		},