Cfg: Add `ImportCallback` to allow caller inspect import status (#536)

* Cfg: Add `ImportCallback` to allow caller inspect import status

.github/workflows/pr.yaml

@@ -22,8 +22,13 @@ jobs:
     steps:
       - name: Checkout Source
         uses: actions/checkout@v3
+
       - name: Run Gosec Security Scanner
-        uses: securego/gosec@2ae137abcf405533ad6e549e9363e58e4f6e8b7d
+        # This is due to https://github.com/securego/gosec/issues/1105
+        # Per https://github.com/securego/gosec/issues/1105#issuecomment-1948225619, the issue occurs since 2.19.0.
+        # The commit that updates the GH action to 2.19.0 is d13d7dac9b7e2b40e86be5b830d297816376f1db
+        # It's parent commit is 26e57d6b340778c2983cd61775bc7e8bb41d002a
+        uses: securego/gosec@26e57d6b340778c2983cd61775bc7e8bb41d002a
         with:
           args: './...'
 

internal/meta/base_meta.go

@@ -92,6 +92,7 @@ type baseMeta struct {
 	providerConfig     map[string]cty.Value
 	fullConfig         bool
 	parallelism        int
+	importCallback     config.ImportCallback
 	generateImportFile bool
 
 	hclOnly  bool
@@ -216,6 +217,7 @@ func NewBaseMeta(cfg config.CommonConfig) (*baseMeta, error) {
 		providerName:       cfg.ProviderName,
 		fullConfig:         cfg.FullConfig,
 		parallelism:        cfg.Parallelism,
+		importCallback:     cfg.ImportCallback,
 		generateImportFile: cfg.GenerateImportBlock,
 		hclOnly:            cfg.HCLOnly,
 		tfclient:           cfg.TFClient,
@@ -273,7 +275,8 @@ func (meta *baseMeta) ParallelImport(ctx context.Context, items []*ImportItem) e
 	meta.tc.Trace(telemetry.Info, "ParallelImport Enter")
 	defer meta.tc.Trace(telemetry.Info, "ParallelImport Leave")
 
-	itemsCh := make(chan *ImportItem, len(items))
+	total := len(items)
+	itemsCh := make(chan *ImportItem, total)
 	for _, item := range items {
 		itemsCh <- item
 	}
@@ -313,6 +316,15 @@ func (meta *baseMeta) ParallelImport(ctx context.Context, items []*ImportItem) e
 		wp.AddTask(func() (interface{}, error) {
 			for item := range itemsCh {
 				meta.importItem(ctx, item, i)
+				if meta.importCallback != nil {
+					item := config.ImportItem{
+						AzureResourceID: item.AzureResourceID,
+						TFResourceId:    item.TFResourceId,
+						ImportError:     item.ImportError,
+						TFAddr:          item.TFAddr,
+					}
+					meta.importCallback(total, item)
+				}
 			}
 			return i, nil
 		})

pkg/config/config.go

@@ -1,13 +1,31 @@
 package config
 
 import (
+	"github.com/Azure/aztfexport/internal/tfaddr"
 	"github.com/Azure/aztfexport/pkg/telemetry"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/arm"
+	"github.com/magodo/armid"
 	"github.com/magodo/terraform-client-go/tfclient"
 	"github.com/zclconf/go-cty/cty"
 )
 
+type ImportItem struct {
+	// Azure resource Id
+	AzureResourceID armid.ResourceId
+
+	// The TF resource id
+	TFResourceId string
+
+	// Whether this azure resource failed to import into terraform (this might due to the TFResourceType doesn't match the resource)
+	ImportError error
+
+	// The terraform resource
+	TFAddr tfaddr.TFAddr
+}
+
+type ImportCallback func(total int, item ImportItem)
+
 type OutputFileNames struct {
 	// The filename for the generated "terraform.tf" (default)
 	TerraformFileName string
@@ -52,6 +70,8 @@ type CommonConfig struct {
 	FullConfig bool
 	// Parallelism specifies the parallelism for the process
 	Parallelism int
+	// ImportCallback is a way to inspect each resource after being imported during ParallelImport
+	ImportCallback ImportCallback
 	// ModulePath specifies the path of the module (e.g. "module1.module2") where the resources will be imported and config generated.
 	// Note that only modules whose "source" is local path is supported. By default, it is the root module.
 	ModulePath string