Adding custom component feature (#1558)
* adding custom component feature
This commit is contained in:
Abhinav Gupta
GitHub
Родитель
46a557ce18
Коммит
0fb27e4086
|
|
@ -1,6 +1,9 @@
|
|||
## 2.4.0 (Unreleased)
|
||||
**Bug Fixes**
|
||||
- [#1426](https://github.com/Azure/azure-storage-fuse/issues/1426) Read panic in block-cache due to boundary conditions.
|
||||
|
||||
**Features**
|
||||
- Added support for custom component via go plugin.
|
||||
|
||||
**Other Changes**
|
||||
- Stream config will be converted to block-cache config implicitly and 'stream' component is no longer used from this release onwards.
|
||||
|
|
|
|||
|
|
@ -201,6 +201,8 @@ To learn about a specific command, just include the name of the command (For exa
|
|||
- CPK options:
|
||||
* `AZURE_STORAGE_CPK_ENCRYPTION_KEY`: Customer provided base64-encoded AES-256 encryption key value.
|
||||
* `AZURE_STORAGE_CPK_ENCRYPTION_KEY_SHA256`: Base64-encoded SHA256 of the cpk encryption key.
|
||||
- Custom component options:
|
||||
* `BLOBFUSE_PLUGIN_PATH`: Specifies plugin file path as a colon-separated list of `.so` files. Example BLOBFUSE_PLUGIN_PATH="/path/to/plugin1.so:/path/to/plugin2.so".
|
||||
|
||||
|
||||
## Config Guide
|
||||
|
|
|
|||
|
|
@ -592,7 +592,7 @@ stages:
|
|||
- script: |
|
||||
echo 'mode: count' > ./blobfuse2_coverage_raw.rpt
|
||||
tail -q -n +2 ./*.cov >> ./blobfuse2_coverage_raw.rpt
|
||||
cat ./blobfuse2_coverage_raw.rpt | grep -v mock_component | grep -v base_component | grep -v loopback | grep -v tools | grep -v "common/log" | grep -v "common/exectime" | grep -v "common/types.go" | grep -v "internal/stats_manager" | grep -v "main.go" | grep -v "component/azstorage/azauthmsi.go" | grep -v "component/azstorage/azauthspn.go" | grep -v "component/stream" | grep -v "component/azstorage/azauthcli.go" > ./blobfuse2_coverage.rpt
|
||||
cat ./blobfuse2_coverage_raw.rpt | grep -v mock_component | grep -v base_component | grep -v loopback | grep -v tools | grep -v "common/log" | grep -v "common/exectime" | grep -v "common/types.go" | grep -v "internal/stats_manager" | grep -v "main.go" | grep -v "component/azstorage/azauthmsi.go" | grep -v "component/azstorage/azauthspn.go" | grep -v "component/stream" | grep -v "component/custom" | grep -v "component/azstorage/azauthcli.go" > ./blobfuse2_coverage.rpt
|
||||
go tool cover -func blobfuse2_coverage.rpt > ./blobfuse2_func_cover.rpt
|
||||
go tool cover -html=./blobfuse2_coverage.rpt -o ./blobfuse2_coverage.html
|
||||
go tool cover -html=./blobfuse2_ut.cov -o ./blobfuse2_ut.html
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ import (
|
|||
_ "github.com/Azure/azure-storage-fuse/v2/component/attr_cache"
|
||||
_ "github.com/Azure/azure-storage-fuse/v2/component/azstorage"
|
||||
_ "github.com/Azure/azure-storage-fuse/v2/component/block_cache"
|
||||
_ "github.com/Azure/azure-storage-fuse/v2/component/custom"
|
||||
_ "github.com/Azure/azure-storage-fuse/v2/component/file_cache"
|
||||
_ "github.com/Azure/azure-storage-fuse/v2/component/libfuse"
|
||||
_ "github.com/Azure/azure-storage-fuse/v2/component/loopback"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,99 @@
|
|||
/*
|
||||
_____ _____ _____ ____ ______ _____ ------
|
||||
| | | | | | | | | | | | |
|
||||
| | | | | | | | | | | | |
|
||||
| --- | | | | |-----| |---- | | |-----| |----- ------
|
||||
| | | | | | | | | | | | |
|
||||
| ____| |_____ | ____| | ____| | |_____| _____| |_____ |_____
|
||||
|
||||
|
||||
Licensed under the MIT License <http://opensource.org/licenses/MIT>.
|
||||
|
||||
Copyright © 2020-2024 Microsoft Corporation. All rights reserved.
|
||||
Author : <blobfusedev@microsoft.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE
|
||||
*/
|
||||
|
||||
package custom
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"plugin"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-storage-fuse/v2/common/log"
|
||||
"github.com/Azure/azure-storage-fuse/v2/exported"
|
||||
"github.com/Azure/azure-storage-fuse/v2/internal"
|
||||
)
|
||||
|
||||
func initializePlugins() error {
|
||||
|
||||
// Environment variable which expects file path as a colon-separated list of `.so` files.
|
||||
// Example BLOBFUSE_PLUGIN_PATH="/path/to/plugin1.so:/path/to/plugin2.so"
|
||||
pluginFilesPath := os.Getenv("BLOBFUSE_PLUGIN_PATH")
|
||||
if pluginFilesPath == "" {
|
||||
log.Info("No plugins to load, BLOBFUSE_PLUGIN_PATH is empty")
|
||||
return nil
|
||||
}
|
||||
|
||||
pluginFiles := strings.Split(pluginFilesPath, ":")
|
||||
|
||||
for _, file := range pluginFiles {
|
||||
if !strings.HasSuffix(file, ".so") {
|
||||
log.Err("Invalid plugin file extension: %s", file)
|
||||
continue
|
||||
}
|
||||
log.Info("loading plugin %s", file)
|
||||
startTime := time.Now()
|
||||
p, err := plugin.Open(file)
|
||||
if err != nil {
|
||||
log.Err("Error opening plugin %s: %s", file, err.Error())
|
||||
return fmt.Errorf("error opening plugin %s: %s", file, err.Error())
|
||||
}
|
||||
|
||||
getExternalComponentFunc, err := p.Lookup("GetExternalComponent")
|
||||
if err != nil {
|
||||
log.Err("GetExternalComponent function lookup error in plugin %s: %s", file, err.Error())
|
||||
return fmt.Errorf("GetExternalComponent function lookup error in plugin %s: %s", file, err.Error())
|
||||
}
|
||||
|
||||
getExternalComponent, ok := getExternalComponentFunc.(func() (string, func() exported.Component))
|
||||
if !ok {
|
||||
log.Err("GetExternalComponent function in %s has some incorrect definition", file)
|
||||
return fmt.Errorf("GetExternalComponent function in %s has some incorrect definition", file)
|
||||
}
|
||||
|
||||
compName, initExternalComponent := getExternalComponent()
|
||||
internal.AddComponent(compName, initExternalComponent)
|
||||
log.Info("Plugin %s loaded in %s", file, time.Since(startTime))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
err := initializePlugins()
|
||||
if err != nil {
|
||||
log.Err("custom::Error initializing plugins: %s", err.Error())
|
||||
fmt.Printf("failed to initialize plugin: %s\n", err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,111 @@
|
|||
/*
|
||||
_____ _____ _____ ____ ______ _____ ------
|
||||
| | | | | | | | | | | | |
|
||||
| | | | | | | | | | | | |
|
||||
| --- | | | | |-----| |---- | | |-----| |----- ------
|
||||
| | | | | | | | | | | | |
|
||||
| ____| |_____ | ____| | ____| | |_____| _____| |_____ |_____
|
||||
|
||||
|
||||
Licensed under the MIT License <http://opensource.org/licenses/MIT>.
|
||||
|
||||
Copyright © 2020-2024 Microsoft Corporation. All rights reserved.
|
||||
Author : <blobfusedev@microsoft.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE
|
||||
*/
|
||||
|
||||
package custom
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
|
||||
type customTestSuite struct {
|
||||
suite.Suite
|
||||
assert *assert.Assertions
|
||||
}
|
||||
|
||||
func (suite *customTestSuite) SetupTest() {
|
||||
suite.assert = assert.New(suite.T())
|
||||
}
|
||||
|
||||
// This test builds a custom component and then tries to load the .so file
|
||||
// Though both .so file and tests are being built using same go packages, it still gives an error in loading .so file
|
||||
//
|
||||
// "plugin was built with a different version of package"
|
||||
//
|
||||
// Hence, this test is disabled for now.
|
||||
// Same .so file loads fine when blobfuse is run from CLI.
|
||||
// If you wish to debug blobfuse2 with a custom component then always build .so file with "-gcflags=all=-N -l" option
|
||||
//
|
||||
// e.g. go build -buildmode=plugin -gcflags=all=-N -l -o x.so <path to source>
|
||||
//
|
||||
// This flag disables all optimizations and inline replacements and then .so will load in debug mode as well.
|
||||
// However same .so will not work with cli mount and there you need to build .so without these flags.
|
||||
func (suite *customTestSuite) _TestInitializePluginsValidPath() {
|
||||
// Direct paths to the Go plugin source files
|
||||
source1 := "../../test/sample_custom_component1/main.go"
|
||||
source2 := "../../test/sample_custom_component2/main.go"
|
||||
|
||||
// Paths to the compiled .so files in the current directory
|
||||
plugin1 := "./sample_custom_component1.so"
|
||||
plugin2 := "./sample_custom_component2.so"
|
||||
|
||||
// Compile the Go plugin source files into .so files
|
||||
cmd := exec.Command("go", "build", "-buildmode=plugin", "-gcflags=all=-N -l", "-o", plugin1, source1)
|
||||
err := cmd.Run()
|
||||
suite.assert.Nil(err)
|
||||
cmd = exec.Command("go", "build", "-buildmode=plugin", "-gcflags=all=-N -l", "-o", plugin2, source2)
|
||||
err = cmd.Run()
|
||||
suite.assert.Nil(err)
|
||||
|
||||
os.Setenv("BLOBFUSE_PLUGIN_PATH", plugin1+":"+plugin2)
|
||||
|
||||
err = initializePlugins()
|
||||
suite.assert.Nil(err)
|
||||
|
||||
// Clean up the generated .so files
|
||||
os.Remove(plugin1)
|
||||
os.Remove(plugin2)
|
||||
}
|
||||
|
||||
func (suite *customTestSuite) TestInitializePluginsInvalidPath() {
|
||||
dummyPath := "/invalid/path/plugin1.so"
|
||||
os.Setenv("BLOBFUSE_PLUGIN_PATH", dummyPath)
|
||||
|
||||
err := initializePlugins()
|
||||
suite.assert.NotNil(err)
|
||||
}
|
||||
|
||||
func (suite *customTestSuite) TestInitializePluginsEmptyPath() {
|
||||
os.Setenv("BLOBFUSE_PLUGIN_PATH", "")
|
||||
|
||||
err := initializePlugins()
|
||||
suite.assert.Nil(err)
|
||||
}
|
||||
|
||||
func TestCustomSuite(t *testing.T) {
|
||||
suite.Run(t, new(customTestSuite))
|
||||
}
|
||||
Загрузка…
Ссылка в новой задаче