From 0fb27e40864763f2c379134fb6e65bab9ebe4cd3 Mon Sep 17 00:00:00 2001 From: Abhinav Gupta <31076415+abhiguptacse@users.noreply.github.com> Date: Tue, 5 Nov 2024 13:10:41 +0530 Subject: [PATCH] Adding custom component feature (#1558) * adding custom component feature --- CHANGELOG.md | 3 + README.md | 2 + blobfuse2-code-coverage.yaml | 2 +- cmd/imports.go | 1 + component/custom/custom.go | 99 ++++++++++++++++++++++++++++ component/custom/custom_test.go | 111 ++++++++++++++++++++++++++++++++ 6 files changed, 217 insertions(+), 1 deletion(-) create mode 100644 component/custom/custom.go create mode 100644 component/custom/custom_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d1b785b..6863d58f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ ## 2.4.0 (Unreleased) **Bug Fixes** - [#1426](https://github.com/Azure/azure-storage-fuse/issues/1426) Read panic in block-cache due to boundary conditions. + +**Features** +- Added support for custom component via go plugin. **Other Changes** - Stream config will be converted to block-cache config implicitly and 'stream' component is no longer used from this release onwards. diff --git a/README.md b/README.md index a7c270c9..0842ff64 100755 --- a/README.md +++ b/README.md @@ -201,6 +201,8 @@ To learn about a specific command, just include the name of the command (For exa - CPK options: * `AZURE_STORAGE_CPK_ENCRYPTION_KEY`: Customer provided base64-encoded AES-256 encryption key value. * `AZURE_STORAGE_CPK_ENCRYPTION_KEY_SHA256`: Base64-encoded SHA256 of the cpk encryption key. +- Custom component options: + * `BLOBFUSE_PLUGIN_PATH`: Specifies plugin file path as a colon-separated list of `.so` files. Example BLOBFUSE_PLUGIN_PATH="/path/to/plugin1.so:/path/to/plugin2.so". ## Config Guide diff --git a/blobfuse2-code-coverage.yaml b/blobfuse2-code-coverage.yaml index 9c9a0299..8783077b 100644 --- a/blobfuse2-code-coverage.yaml +++ b/blobfuse2-code-coverage.yaml @@ -592,7 +592,7 @@ stages: - script: | echo 'mode: count' > ./blobfuse2_coverage_raw.rpt tail -q -n +2 ./*.cov >> ./blobfuse2_coverage_raw.rpt - cat ./blobfuse2_coverage_raw.rpt | grep -v mock_component | grep -v base_component | grep -v loopback | grep -v tools | grep -v "common/log" | grep -v "common/exectime" | grep -v "common/types.go" | grep -v "internal/stats_manager" | grep -v "main.go" | grep -v "component/azstorage/azauthmsi.go" | grep -v "component/azstorage/azauthspn.go" | grep -v "component/stream" | grep -v "component/azstorage/azauthcli.go" > ./blobfuse2_coverage.rpt + cat ./blobfuse2_coverage_raw.rpt | grep -v mock_component | grep -v base_component | grep -v loopback | grep -v tools | grep -v "common/log" | grep -v "common/exectime" | grep -v "common/types.go" | grep -v "internal/stats_manager" | grep -v "main.go" | grep -v "component/azstorage/azauthmsi.go" | grep -v "component/azstorage/azauthspn.go" | grep -v "component/stream" | grep -v "component/custom" | grep -v "component/azstorage/azauthcli.go" > ./blobfuse2_coverage.rpt go tool cover -func blobfuse2_coverage.rpt > ./blobfuse2_func_cover.rpt go tool cover -html=./blobfuse2_coverage.rpt -o ./blobfuse2_coverage.html go tool cover -html=./blobfuse2_ut.cov -o ./blobfuse2_ut.html diff --git a/cmd/imports.go b/cmd/imports.go index 9178cb71..0e576733 100644 --- a/cmd/imports.go +++ b/cmd/imports.go @@ -37,6 +37,7 @@ import ( _ "github.com/Azure/azure-storage-fuse/v2/component/attr_cache" _ "github.com/Azure/azure-storage-fuse/v2/component/azstorage" _ "github.com/Azure/azure-storage-fuse/v2/component/block_cache" + _ "github.com/Azure/azure-storage-fuse/v2/component/custom" _ "github.com/Azure/azure-storage-fuse/v2/component/file_cache" _ "github.com/Azure/azure-storage-fuse/v2/component/libfuse" _ "github.com/Azure/azure-storage-fuse/v2/component/loopback" diff --git a/component/custom/custom.go b/component/custom/custom.go new file mode 100644 index 00000000..d95f4661 --- /dev/null +++ b/component/custom/custom.go @@ -0,0 +1,99 @@ +/* + _____ _____ _____ ____ ______ _____ ------ + | | | | | | | | | | | | | + | | | | | | | | | | | | | + | --- | | | | |-----| |---- | | |-----| |----- ------ + | | | | | | | | | | | | | + | ____| |_____ | ____| | ____| | |_____| _____| |_____ |_____ + + + Licensed under the MIT License . + + Copyright © 2020-2024 Microsoft Corporation. All rights reserved. + Author : + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE +*/ + +package custom + +import ( + "fmt" + "os" + "plugin" + "strings" + "time" + + "github.com/Azure/azure-storage-fuse/v2/common/log" + "github.com/Azure/azure-storage-fuse/v2/exported" + "github.com/Azure/azure-storage-fuse/v2/internal" +) + +func initializePlugins() error { + + // Environment variable which expects file path as a colon-separated list of `.so` files. + // Example BLOBFUSE_PLUGIN_PATH="/path/to/plugin1.so:/path/to/plugin2.so" + pluginFilesPath := os.Getenv("BLOBFUSE_PLUGIN_PATH") + if pluginFilesPath == "" { + log.Info("No plugins to load, BLOBFUSE_PLUGIN_PATH is empty") + return nil + } + + pluginFiles := strings.Split(pluginFilesPath, ":") + + for _, file := range pluginFiles { + if !strings.HasSuffix(file, ".so") { + log.Err("Invalid plugin file extension: %s", file) + continue + } + log.Info("loading plugin %s", file) + startTime := time.Now() + p, err := plugin.Open(file) + if err != nil { + log.Err("Error opening plugin %s: %s", file, err.Error()) + return fmt.Errorf("error opening plugin %s: %s", file, err.Error()) + } + + getExternalComponentFunc, err := p.Lookup("GetExternalComponent") + if err != nil { + log.Err("GetExternalComponent function lookup error in plugin %s: %s", file, err.Error()) + return fmt.Errorf("GetExternalComponent function lookup error in plugin %s: %s", file, err.Error()) + } + + getExternalComponent, ok := getExternalComponentFunc.(func() (string, func() exported.Component)) + if !ok { + log.Err("GetExternalComponent function in %s has some incorrect definition", file) + return fmt.Errorf("GetExternalComponent function in %s has some incorrect definition", file) + } + + compName, initExternalComponent := getExternalComponent() + internal.AddComponent(compName, initExternalComponent) + log.Info("Plugin %s loaded in %s", file, time.Since(startTime)) + } + return nil +} + +func init() { + err := initializePlugins() + if err != nil { + log.Err("custom::Error initializing plugins: %s", err.Error()) + fmt.Printf("failed to initialize plugin: %s\n", err.Error()) + os.Exit(1) + } +} diff --git a/component/custom/custom_test.go b/component/custom/custom_test.go new file mode 100644 index 00000000..c4ab82df --- /dev/null +++ b/component/custom/custom_test.go @@ -0,0 +1,111 @@ +/* + _____ _____ _____ ____ ______ _____ ------ + | | | | | | | | | | | | | + | | | | | | | | | | | | | + | --- | | | | |-----| |---- | | |-----| |----- ------ + | | | | | | | | | | | | | + | ____| |_____ | ____| | ____| | |_____| _____| |_____ |_____ + + + Licensed under the MIT License . + + Copyright © 2020-2024 Microsoft Corporation. All rights reserved. + Author : + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE +*/ + +package custom + +import ( + "os" + "os/exec" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type customTestSuite struct { + suite.Suite + assert *assert.Assertions +} + +func (suite *customTestSuite) SetupTest() { + suite.assert = assert.New(suite.T()) +} + +// This test builds a custom component and then tries to load the .so file +// Though both .so file and tests are being built using same go packages, it still gives an error in loading .so file +// +// "plugin was built with a different version of package" +// +// Hence, this test is disabled for now. +// Same .so file loads fine when blobfuse is run from CLI. +// If you wish to debug blobfuse2 with a custom component then always build .so file with "-gcflags=all=-N -l" option +// +// e.g. go build -buildmode=plugin -gcflags=all=-N -l -o x.so +// +// This flag disables all optimizations and inline replacements and then .so will load in debug mode as well. +// However same .so will not work with cli mount and there you need to build .so without these flags. +func (suite *customTestSuite) _TestInitializePluginsValidPath() { + // Direct paths to the Go plugin source files + source1 := "../../test/sample_custom_component1/main.go" + source2 := "../../test/sample_custom_component2/main.go" + + // Paths to the compiled .so files in the current directory + plugin1 := "./sample_custom_component1.so" + plugin2 := "./sample_custom_component2.so" + + // Compile the Go plugin source files into .so files + cmd := exec.Command("go", "build", "-buildmode=plugin", "-gcflags=all=-N -l", "-o", plugin1, source1) + err := cmd.Run() + suite.assert.Nil(err) + cmd = exec.Command("go", "build", "-buildmode=plugin", "-gcflags=all=-N -l", "-o", plugin2, source2) + err = cmd.Run() + suite.assert.Nil(err) + + os.Setenv("BLOBFUSE_PLUGIN_PATH", plugin1+":"+plugin2) + + err = initializePlugins() + suite.assert.Nil(err) + + // Clean up the generated .so files + os.Remove(plugin1) + os.Remove(plugin2) +} + +func (suite *customTestSuite) TestInitializePluginsInvalidPath() { + dummyPath := "/invalid/path/plugin1.so" + os.Setenv("BLOBFUSE_PLUGIN_PATH", dummyPath) + + err := initializePlugins() + suite.assert.NotNil(err) +} + +func (suite *customTestSuite) TestInitializePluginsEmptyPath() { + os.Setenv("BLOBFUSE_PLUGIN_PATH", "") + + err := initializePlugins() + suite.assert.Nil(err) +} + +func TestCustomSuite(t *testing.T) { + suite.Run(t, new(customTestSuite)) +}