Removing usage of SPN in pipelines (#1431)
* Removing usage of SPN in pipelines
This commit is contained in:
Vikas Bhansali
GitHub
Родитель
0790773436
Коммит
cefb0226f9
|
|
@ -35,13 +35,14 @@ jobs:
|
|||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v1
|
||||
uses: github/codeql-action/init@v3
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
|
||||
# If you wish to specify custom queries, you can do so here or in a config file.
|
||||
# By default, queries listed here will override any specified in a config file.
|
||||
# Prefix the list here with "+" to use these queries and those in the config file.
|
||||
|
|
@ -50,7 +51,7 @@ jobs:
|
|||
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
|
||||
# If this step fails, then you should remove it and run the build manually (see below)
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@v1
|
||||
uses: github/codeql-action/autobuild@v3
|
||||
|
||||
# ℹ️ Command-line programs to run using the OS shell.
|
||||
# 📚 https://git.io/JvXDl
|
||||
|
|
@ -64,4 +65,4 @@ jobs:
|
|||
# make release
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v1
|
||||
uses: github/codeql-action/analyze@v3
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ steps:
|
|||
|
||||
# azcli login
|
||||
- script: |
|
||||
az login --service-principal -u $(AZTEST_CLIENT) -p $(AZTEST_SECRET) --tenant $(AZTEST_TENANT)
|
||||
az login --identity --username $(ACLI_BLOBFUSE_MSI_APP_ID)
|
||||
displayName: 'Azure CLI login'
|
||||
|
||||
- task: Go@0
|
||||
|
|
@ -74,9 +74,6 @@ steps:
|
|||
echo "\"msi-appid\"": "\"$(AZTEST_APP_ID)\"", >> $cnfFile
|
||||
echo "\"msi-resid\"": "\"$(AZTEST_RES_ID)\"", >> $cnfFile
|
||||
echo "\"msi-objid\"": "\"$(AZTEST_OBJ_ID)\"", >> $cnfFile
|
||||
echo "\"spn-client\"": "\"$(AZTEST_CLIENT)\"", >> $cnfFile
|
||||
echo "\"spn-tenant\"": "\"$(AZTEST_TENANT)\"", >> $cnfFile
|
||||
echo "\"spn-secret\"": "\"$(AZTEST_SECRET)\"", >> $cnfFile
|
||||
echo "\"skip-msi\"": "${{ parameters.skip_msi }}", >> $cnfFile
|
||||
echo "\"skip-azcli\"": "false", >> $cnfFile
|
||||
echo "\"proxy-address\"": "\"${{ parameters.proxy_address }}\"" >> $cnfFile
|
||||
|
|
|
|||
|
|
@ -80,9 +80,6 @@ steps:
|
|||
echo "\"msi-appid\"": "\"$(AZTEST_APP_ID)\"", >> $cnfFile
|
||||
echo "\"msi-resid\"": "\"$(AZTEST_RES_ID)\"", >> $cnfFile
|
||||
echo "\"msi-objid\"": "\"$(AZTEST_OBJ_ID)\"", >> $cnfFile
|
||||
echo "\"spn-client\"": "\"$(AZTEST_CLIENT)\"", >> $cnfFile
|
||||
echo "\"spn-tenant\"": "\"$(AZTEST_TENANT)\"", >> $cnfFile
|
||||
echo "\"spn-secret\"": "\"$(AZTEST_SECRET)\"", >> $cnfFile
|
||||
echo "\"skip-msi\"": "true", >> $cnfFile
|
||||
echo "\"skip-azcli\"": "true", >> $cnfFile
|
||||
echo "\"proxy-address\"": "\"\"" >> $cnfFile
|
||||
|
|
|
|||
|
|
@ -124,9 +124,6 @@ steps:
|
|||
echo "\"msi-appid\"": "\"$(AZTEST_APP_ID)\"", >> $cnfFile
|
||||
echo "\"msi-resid\"": "\"$(AZTEST_RES_ID)\"", >> $cnfFile
|
||||
echo "\"msi-objid\"": "\"$(AZTEST_OBJ_ID)\"", >> $cnfFile
|
||||
echo "\"spn-client\"": "\"$(AZTEST_CLIENT)\"", >> $cnfFile
|
||||
echo "\"spn-tenant\"": "\"$(AZTEST_TENANT)\"", >> $cnfFile
|
||||
echo "\"spn-secret\"": "\"$(AZTEST_SECRET)\"", >> $cnfFile
|
||||
echo "\"skip-msi\"": "${{ parameters.skip_msi }}", >> $cnfFile
|
||||
echo "\"skip-azcli\"": "${{ parameters.skip_azcli }}", >> $cnfFile
|
||||
echo "\"proxy-address\"": "\"${{ parameters.proxy_address }}\"" >> $cnfFile
|
||||
|
|
@ -146,7 +143,7 @@ steps:
|
|||
|
||||
# azcli login
|
||||
- script: |
|
||||
az login --service-principal -u $(AZTEST_CLIENT) -p $(AZTEST_SECRET) --tenant $(AZTEST_TENANT)
|
||||
az login --identity --username $(ACLI_BLOBFUSE_MSI_APP_ID)
|
||||
displayName: 'Azure CLI login'
|
||||
condition: eq('${{ parameters.skip_azcli }}', 'false')
|
||||
|
||||
|
|
|
|||
|
|
@ -42,8 +42,6 @@ parameters:
|
|||
default: false
|
||||
|
||||
steps:
|
||||
- checkout: none
|
||||
|
||||
# Get the host details on which these test are running
|
||||
- script: |
|
||||
echo $(Description)
|
||||
|
|
@ -52,6 +50,9 @@ steps:
|
|||
|
||||
# Create directory structure and prepare to mount
|
||||
- ${{ parameters.installStep }}
|
||||
|
||||
- checkout: none
|
||||
|
||||
- script: |
|
||||
sudo rm -rf $(ROOT_DIR)
|
||||
sudo mkdir -p $(ROOT_DIR)
|
||||
|
|
@ -101,7 +102,6 @@ steps:
|
|||
quick_test: ${{ parameters.quick_test }}
|
||||
verbose_log: ${{ parameters.verbose_log }}
|
||||
clone: ${{ parameters.clone }}
|
||||
stream_direct_test: false
|
||||
# TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable
|
||||
temp_dir: $(TEMP_DIR)
|
||||
mount_dir: $(MOUNT_DIR)
|
||||
|
|
@ -122,7 +122,6 @@ steps:
|
|||
quick_test: ${{ parameters.quick_test }}
|
||||
verbose_log: ${{ parameters.verbose_log }}
|
||||
clone: ${{ parameters.clone }}
|
||||
stream_direct_test: false
|
||||
# TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable
|
||||
temp_dir: $(TEMP_DIR)
|
||||
mount_dir: $(MOUNT_DIR)
|
||||
|
|
|
|||
|
|
@ -31,9 +31,6 @@ parameters:
|
|||
- name: clone
|
||||
type: boolean
|
||||
default: false
|
||||
- name: stream_direct_test
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
steps:
|
||||
- script: |
|
||||
|
|
|
|||
|
|
@ -31,9 +31,6 @@ parameters:
|
|||
- name: clone
|
||||
type: boolean
|
||||
default: false
|
||||
- name: stream_direct_test
|
||||
type: boolean
|
||||
default: false
|
||||
- name: enable_symlink_adls
|
||||
type: boolean
|
||||
default: false
|
||||
|
|
@ -67,7 +64,6 @@ steps:
|
|||
artifact_name: '${{ parameters.distro_name }}_${{ parameters.idstring }}.txt'
|
||||
verbose_log: ${{ parameters.verbose_log }}
|
||||
clone: ${{ parameters.clone }}
|
||||
stream_direct_test: ${{ parameters.stream_direct_test }}
|
||||
enable_symlink_adls: ${{ parameters.enable_symlink_adls }}
|
||||
mountStep:
|
||||
script: |
|
||||
|
|
|
|||
|
|
@ -19,9 +19,6 @@ parameters:
|
|||
- name: quick_test
|
||||
type: boolean
|
||||
default: true
|
||||
- name: stream_direct_test
|
||||
type: boolean
|
||||
default: false
|
||||
- name: enable_symlink_adls
|
||||
type: boolean
|
||||
default: false
|
||||
|
|
@ -69,7 +66,7 @@ steps:
|
|||
- task: Go@0
|
||||
inputs:
|
||||
command: 'test'
|
||||
arguments: '-v -timeout=2h ./... -args -mnt-path=${{ parameters.mount_dir }} -adls=${{parameters.adls}} -clone=${{parameters.clone}} -tmp-path=${{parameters.temp_dir}} -quick-test=${{parameters.quick_test}} -stream-direct-test=${{parameters.stream_direct_test}} -enable-symlink-adls=${{parameters.enable_symlink_adls}} -distro-name="${{parameters.distro_name}}"'
|
||||
arguments: '-v -timeout=2h ./... -args -mnt-path=${{ parameters.mount_dir }} -adls=${{parameters.adls}} -clone=${{parameters.clone}} -tmp-path=${{parameters.temp_dir}} -quick-test=${{parameters.quick_test}} -enable-symlink-adls=${{parameters.enable_symlink_adls}} -distro-name="${{parameters.distro_name}}"'
|
||||
workingDirectory: ${{ parameters.working_dir }}/test/e2e_tests
|
||||
displayName: 'E2E Test: ${{ parameters.idstring }}'
|
||||
timeoutInMinutes: 120
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ parameters:
|
|||
default: "null"
|
||||
|
||||
steps:
|
||||
# Create directory structure and prepare to mount
|
||||
- ${{ parameters.installStep }}
|
||||
- checkout: none
|
||||
|
||||
# Get the host details on which these test are running
|
||||
|
|
@ -14,8 +16,6 @@ steps:
|
|||
hostnamectl
|
||||
displayName: 'Print Agent Info'
|
||||
|
||||
# Create directory structure and prepare to mount
|
||||
- ${{ parameters.installStep }}
|
||||
- script: |
|
||||
sudo rm -rf $(ROOT_DIR)
|
||||
sudo mkdir -p $(ROOT_DIR)
|
||||
|
|
|
|||
|
|
@ -5,14 +5,10 @@ parameters:
|
|||
type: string
|
||||
- name: account_endpoint
|
||||
type: string
|
||||
- name: spn_account_endpoint
|
||||
type: string
|
||||
- name: adls
|
||||
type: boolean
|
||||
- name: account_name
|
||||
type: string
|
||||
- name: spn_account_name
|
||||
type: string
|
||||
- name: account_key
|
||||
type: string
|
||||
- name: account_sas
|
||||
|
|
@ -43,23 +39,13 @@ parameters:
|
|||
type: boolean
|
||||
- name: test_sas_credential
|
||||
type: boolean
|
||||
- name: test_spn_credential
|
||||
type: boolean
|
||||
- name: test_azcli_credential
|
||||
type: boolean
|
||||
- name: test_stream
|
||||
type: boolean
|
||||
- name: stream_config
|
||||
type: string
|
||||
- name: stream_filename_config
|
||||
type: string
|
||||
- name: test_azurite
|
||||
type: boolean
|
||||
default: false
|
||||
- name: sas_credential_config
|
||||
type: string
|
||||
- name: spn_credential_config
|
||||
type: string
|
||||
- name: azcli_credential_config
|
||||
type: string
|
||||
- name: azurite_config
|
||||
|
|
@ -96,42 +82,6 @@ steps:
|
|||
displayName: Print config file
|
||||
condition: ${{ parameters.test_key_credential }}
|
||||
|
||||
# Stream e2e
|
||||
- script: |
|
||||
cd ${{ parameters.working_dir }}
|
||||
${{ parameters.working_dir }}/blobfuse2 gen-test-config --config-file=azure_stream.yaml --container-name=${{ parameters.container }} --output-file=${{ parameters.stream_config }}
|
||||
displayName: Create Stream Config File
|
||||
env:
|
||||
ACCOUNT_TYPE: ${{ parameters.account_type }}
|
||||
NIGHTLY_STO_ACC_NAME: ${{ parameters.account_name }}
|
||||
NIGHTLY_STO_ACC_KEY: ${{ parameters.account_key }}
|
||||
ACCOUNT_ENDPOINT: ${{ parameters.account_endpoint }}
|
||||
VERBOSE_LOG: ${{ parameters.verbose_log }}
|
||||
condition: ${{ parameters.test_stream }}
|
||||
continueOnError: false
|
||||
|
||||
- script: cat ${{ parameters.stream_config }}
|
||||
displayName: Print Stream config file with Handle Level Caching
|
||||
condition: ${{ parameters.test_stream }}
|
||||
|
||||
# Stream e2e filename level caching
|
||||
- script: |
|
||||
cd ${{ parameters.working_dir }}
|
||||
${{ parameters.working_dir }}/blobfuse2 gen-test-config --config-file=azure_stream_filename.yaml --container-name=${{ parameters.container }} --output-file=${{ parameters.stream_filename_config }}
|
||||
displayName: Create Stream Config File
|
||||
env:
|
||||
ACCOUNT_TYPE: ${{ parameters.account_type }}
|
||||
NIGHTLY_STO_ACC_NAME: ${{ parameters.account_name }}
|
||||
NIGHTLY_STO_ACC_KEY: ${{ parameters.account_key }}
|
||||
ACCOUNT_ENDPOINT: ${{ parameters.account_endpoint }}
|
||||
VERBOSE_LOG: ${{ parameters.verbose_log }}
|
||||
condition: ${{ parameters.test_stream }}
|
||||
continueOnError: false
|
||||
|
||||
- script: cat ${{ parameters.stream_filename_config }}
|
||||
displayName: Print Stream config file with Filename Caching
|
||||
condition: ${{ parameters.test_stream }}
|
||||
|
||||
# Create sas credential config file if we need to test it
|
||||
- script: |
|
||||
cd ${{ parameters.working_dir }}
|
||||
|
|
@ -150,33 +100,13 @@ steps:
|
|||
displayName: Print SAS config file
|
||||
condition: ${{ parameters.test_sas_credential }}
|
||||
|
||||
# Create spn credential config file if we need to test it
|
||||
- script: |
|
||||
cd ${{ parameters.working_dir }}
|
||||
${{ parameters.working_dir }}/blobfuse2 gen-test-config --config-file=azure_spn.yaml --container-name=${{ parameters.container }} --temp-path=${{ parameters.temp_dir }} --output-file=${{ parameters.spn_credential_config }}
|
||||
displayName: Create SPN Config File
|
||||
env:
|
||||
NIGHTLY_SPN_ACC_NAME: ${{ parameters.spn_account_name }}
|
||||
NIGHTLY_SPN_CLIENT_ID: ${{ parameters.client_id }}
|
||||
NIGHTLY_SPN_TENANT_ID: ${{ parameters.tenant_id }}
|
||||
NIGHTLY_SPN_CLIENT_SECRET: ${{ parameters.client_secret }}
|
||||
ACCOUNT_TYPE: ${{ parameters.account_type }}
|
||||
ACCOUNT_ENDPOINT: ${{ parameters.spn_account_endpoint }}
|
||||
VERBOSE_LOG: ${{ parameters.verbose_log }}
|
||||
condition: ${{ parameters.test_spn_credential }}
|
||||
continueOnError: false
|
||||
|
||||
- script: cat ${{ parameters.spn_credential_config }}
|
||||
displayName: Print SPN config file
|
||||
condition: ${{ parameters.test_spn_credential }}
|
||||
|
||||
# Create azcli credential config file if we need to test it
|
||||
- script: |
|
||||
cd ${{ parameters.working_dir }}
|
||||
${{ parameters.working_dir }}/blobfuse2 gen-test-config --config-file=azure_cli.yaml --container-name=${{ parameters.container }} --temp-path=${{ parameters.temp_dir }} --output-file=${{ parameters.azcli_credential_config }}
|
||||
displayName: Create Azure CLI Config File
|
||||
env:
|
||||
NIGHTLY_STO_BLOB_ACC_NAME: ${{ parameters.spn_account_name }}
|
||||
NIGHTLY_STO_BLOB_ACC_NAME: ${{ parameters.account_name }}
|
||||
ACCOUNT_TYPE: ${{ parameters.account_type }}
|
||||
ACCOUNT_ENDPOINT: ${{ parameters.account_endpoint }}
|
||||
VERBOSE_LOG: ${{ parameters.verbose_log }}
|
||||
|
|
@ -246,47 +176,6 @@ steps:
|
|||
quick_test: ${{ parameters.quick_test }}
|
||||
verbose_log: ${{ parameters.verbose_log }}
|
||||
clone: false
|
||||
stream_direct_test: false
|
||||
|
||||
- ${{ if eq(parameters.test_stream, true) }}:
|
||||
- template: e2e-tests.yml
|
||||
parameters:
|
||||
working_dir: ${{ parameters.working_dir }}
|
||||
temp_dir: ${{ parameters.temp_dir }}
|
||||
mount_dir: ${{ parameters.mount_dir }}
|
||||
adls: ${{ parameters.adls }}
|
||||
idstring: ${{ parameters.service }} with Streaming
|
||||
distro_name: ${{ parameters.distro_name }}
|
||||
quick_test: ${{ parameters.quick_test }}
|
||||
artifact_name: '${{ parameters.distro_name }}_${{ parameters.service }}_stream.txt'
|
||||
verbose_log: ${{ parameters.verbose_log }}
|
||||
mountStep:
|
||||
script: >
|
||||
${{ parameters.working_dir }}/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.stream_config }}
|
||||
--default-working-dir=${{ parameters.working_dir }}
|
||||
displayName: 'E2E Test: Mount with Stream Configuration'
|
||||
timeoutInMinutes: 3
|
||||
continueOnError: false
|
||||
|
||||
- ${{ if eq(parameters.test_stream, true) }}:
|
||||
- template: e2e-tests.yml
|
||||
parameters:
|
||||
working_dir: ${{ parameters.working_dir }}
|
||||
temp_dir: ${{ parameters.temp_dir }}
|
||||
mount_dir: ${{ parameters.mount_dir }}
|
||||
adls: ${{ parameters.adls }}
|
||||
idstring: ${{ parameters.service }} with Streaming with filename
|
||||
distro_name: ${{ parameters.distro_name }}
|
||||
quick_test: ${{ parameters.quick_test }}
|
||||
artifact_name: '${{ parameters.distro_name }}_${{ parameters.service }}_stream_with_filename.txt'
|
||||
verbose_log: ${{ parameters.verbose_log }}
|
||||
mountStep:
|
||||
script: >
|
||||
${{ parameters.working_dir }}/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.stream_filename_config }}
|
||||
--default-working-dir=${{ parameters.working_dir }}
|
||||
displayName: 'E2E Test: Mount with Filename Stream Configuration'
|
||||
timeoutInMinutes: 3
|
||||
continueOnError: false
|
||||
|
||||
- ${{ if eq(parameters.test_sas_credential, true) }}:
|
||||
- template: e2e-tests.yml
|
||||
|
|
@ -307,25 +196,6 @@ steps:
|
|||
timeoutInMinutes: 3
|
||||
continueOnError: false
|
||||
|
||||
- ${{ if eq(parameters.test_spn_credential, true) }}:
|
||||
- template: e2e-tests.yml
|
||||
parameters:
|
||||
working_dir: ${{ parameters.working_dir }}
|
||||
mount_dir: ${{ parameters.mount_dir }}
|
||||
temp_dir: ${{ parameters.temp_dir }}
|
||||
adls: ${{ parameters.adls }}
|
||||
idstring: ${{ parameters.service }} with SPN Credentials
|
||||
distro_name: ${{ parameters.distro_name }}
|
||||
artifact_name: '${{ parameters.distro_name }}_${{ parameters.service }}_spn.txt'
|
||||
verbose_log: ${{ parameters.verbose_log }}
|
||||
mountStep:
|
||||
script: >
|
||||
${{ parameters.working_dir }}/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.spn_credential_config }}
|
||||
--default-working-dir=${{ parameters.working_dir }}
|
||||
displayName: 'E2E Test: Mount with SPN Credential Configuration'
|
||||
timeoutInMinutes: 3
|
||||
continueOnError: false
|
||||
|
||||
- ${{ if eq(parameters.test_azcli_credential, true) }}:
|
||||
- template: e2e-tests.yml
|
||||
parameters:
|
||||
|
|
@ -347,8 +217,8 @@ steps:
|
|||
|
||||
- ${{ if eq(parameters.test_azurite, true) }}:
|
||||
- bash: |
|
||||
sudo apt-get install azure-cli -y
|
||||
sudo npm install -g azurite
|
||||
sudo apt-get install azure-cli npm -y
|
||||
sudo npm install -g azurite@3.29.0
|
||||
sudo mkdir azurite
|
||||
sudo azurite --silent --location azurite --debug azurite\debug.log &
|
||||
az storage container create -n ${{ parameters.container }} --connection-string "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;"
|
||||
|
|
@ -375,22 +245,6 @@ steps:
|
|||
|
||||
#--------------------------------------- Tests: End to end tests with different File Cache configurations ------------------------------------------
|
||||
|
||||
- template: e2e-tests-spcl.yml
|
||||
parameters:
|
||||
conf_template: azure_key_lfu.yaml
|
||||
config_file: ${{ parameters.config }}
|
||||
container: ${{ parameters.container }}
|
||||
temp_dir: ${{ parameters.temp_dir }}
|
||||
mount_dir: ${{ parameters.mount_dir }}
|
||||
adls: ${{ parameters.adls }}
|
||||
account_name: ${{ parameters.account_name }}
|
||||
account_key: ${{ parameters.account_key }}
|
||||
account_type: ${{ parameters.account_type }}
|
||||
account_endpoint: ${{ parameters.account_endpoint }}
|
||||
idstring: "${{ parameters.service }} LFU policy"
|
||||
distro_name: ${{ parameters.distro_name }}
|
||||
verbose_log: ${{ parameters.verbose_log }}
|
||||
|
||||
- template: e2e-tests-spcl.yml
|
||||
parameters:
|
||||
conf_template: azure_key_lru_purge.yaml
|
||||
|
|
|
|||
|
|
@ -13,23 +13,26 @@ jobs:
|
|||
strategy:
|
||||
matrix:
|
||||
Ubuntu-20:
|
||||
imageName: 'ubuntu-20.04'
|
||||
AgentName: 'blobfuse-ubuntu20'
|
||||
containerName: 'test-cnt-ubn-20'
|
||||
fuselib: 'libfuse-dev'
|
||||
tags: 'fuse2'
|
||||
adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_20)
|
||||
Ubuntu-22:
|
||||
imageName: 'ubuntu-22.04'
|
||||
AgentName: 'blobfuse-ubuntu22'
|
||||
containerName: 'test-cnt-ubn-22'
|
||||
fuselib: 'libfuse3-dev'
|
||||
tags: 'fuse3'
|
||||
adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22)
|
||||
|
||||
pool:
|
||||
vmImage: $(imageName)
|
||||
name: "blobfuse-ubuntu-pool"
|
||||
demands:
|
||||
- ImageOverride -equals $(AgentName)
|
||||
|
||||
variables:
|
||||
- group: NightlyBlobFuse
|
||||
|
||||
|
||||
steps:
|
||||
# ----------------------------------------------------------------
|
||||
- template: 'azure-pipeline-templates/blobfuse2-ci-template.yml'
|
||||
|
|
@ -44,7 +47,7 @@ jobs:
|
|||
strategy:
|
||||
matrix:
|
||||
Ubuntu-22-ARM64:
|
||||
imageName: 'blobfuse-ubn22-arm64'
|
||||
AgentName: 'blobfuse-ubn22-arm64'
|
||||
containerName: 'test-cnt-ubn-22'
|
||||
fuselib: 'libfuse3-dev'
|
||||
tags: 'fuse3'
|
||||
|
|
@ -53,7 +56,7 @@ jobs:
|
|||
pool:
|
||||
name: "blobfuse-ubn-arm64-pool"
|
||||
demands:
|
||||
- ImageOverride -equals $(imageName)
|
||||
- ImageOverride -equals $(AgentName)
|
||||
|
||||
variables:
|
||||
- group: NightlyBlobFuse
|
||||
|
|
|
|||
|
|
@ -28,14 +28,12 @@ stages:
|
|||
matrix:
|
||||
Ubuntu-20:
|
||||
AgentName: 'blobfuse-ubuntu20'
|
||||
imageName: 'ubuntu-20.04'
|
||||
containerName: 'test-cnt-ubn-20'
|
||||
fuselib: 'libfuse-dev'
|
||||
fuselib2: 'fuse'
|
||||
tags: 'fuse2'
|
||||
Ubuntu-22:
|
||||
AgentName: 'blobfuse-ubuntu22'
|
||||
imageName: 'ubuntu-22.04'
|
||||
containerName: 'test-cnt-ubn-22'
|
||||
fuselib: 'libfuse3-dev'
|
||||
fuselib2: 'fuse3'
|
||||
|
|
@ -54,8 +52,6 @@ stages:
|
|||
value: '$(Pipeline.Workspace)/blobfuse2_tmp'
|
||||
- name: BLOBFUSE2_CFG
|
||||
value: '$(Pipeline.Workspace)/blobfuse2.yaml'
|
||||
- name: BLOBFUSE2_STREAM_CFG
|
||||
value: '$(Pipeline.Workspace)/blobfuse2_stream.yaml'
|
||||
- name: BLOBFUSE2_ADLS_CFG
|
||||
value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml'
|
||||
- name: GOPATH
|
||||
|
|
@ -236,84 +232,6 @@ stages:
|
|||
workingDirectory: $(WORK_DIR)
|
||||
displayName: "ADLS Coverage with profilers"
|
||||
|
||||
|
||||
# -------------------------------------------------------
|
||||
# Config Generation (Block Blob - LFU policy)
|
||||
- script: |
|
||||
cd $(WORK_DIR)
|
||||
./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/blobfuse2_gentest2.cov gen-test-config --config-file=azure_key_lfu.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG)
|
||||
env:
|
||||
NIGHTLY_STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME)
|
||||
NIGHTLY_STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY)
|
||||
ACCOUNT_TYPE: 'block'
|
||||
ACCOUNT_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net'
|
||||
VERBOSE_LOG: false
|
||||
displayName: 'Create Config File - LFU'
|
||||
continueOnError: false
|
||||
workingDirectory: $(WORK_DIR)
|
||||
|
||||
# Code Coverage with e2e-tests for block blob with lfu policy
|
||||
- script: |
|
||||
rm -rf $(MOUNT_DIR)/*
|
||||
rm -rf $(TEMP_DIR)/*
|
||||
./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/blobfuse2_block_lfu.cov mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --foreground=true &
|
||||
sleep 10
|
||||
ps -aux | grep blobfuse2
|
||||
rm -rf $(MOUNT_DIR)/*
|
||||
cd test/e2e_tests
|
||||
go test -v -timeout=7200s ./... -args -mnt-path=$(MOUNT_DIR) -tmp-path=$(TEMP_DIR)
|
||||
cd -
|
||||
./blobfuse2 unmount $(MOUNT_DIR)
|
||||
sleep 5
|
||||
workingDirectory: $(WORK_DIR)
|
||||
displayName: "Block Blob LFU Coverage"
|
||||
|
||||
|
||||
# -------------------------------------------------------
|
||||
# Config Generation (Block Blob - Stream)
|
||||
- script: |
|
||||
cd $(WORK_DIR)
|
||||
./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/blobfuse2_gentest3.cov gen-test-config --config-file=azure_stream.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_STREAM_CFG)
|
||||
displayName: 'Create Config File - Stream'
|
||||
env:
|
||||
NIGHTLY_STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME)
|
||||
NIGHTLY_STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY)
|
||||
ACCOUNT_TYPE: 'block'
|
||||
ACCOUNT_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net'
|
||||
VERBOSE_LOG: false
|
||||
continueOnError: false
|
||||
workingDirectory: $(WORK_DIR)
|
||||
|
||||
# Streaming test preparation
|
||||
- script: |
|
||||
rm -rf $(MOUNT_DIR)/*
|
||||
rm -rf $(TEMP_DIR)/*
|
||||
./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/blobfuse2_stream_prep.cov mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --foreground=true &
|
||||
sleep 10
|
||||
ps -aux | grep blobfuse2
|
||||
for i in {10,50,100,200,500,1024}; do echo $i; done | parallel --will-cite -j 5 'head -c {}M < /dev/urandom > $(WORK_DIR)/myfile_{}'
|
||||
for i in {10,50,100,200,500,1024}; do echo $i; done | parallel --will-cite -j 5 'cp $(WORK_DIR)/myfile_{} $(MOUNT_DIR)/'
|
||||
./blobfuse2 unmount "$(MOUNT_DIR)*"
|
||||
sudo fusermount -u $(MOUNT_DIR)
|
||||
sleep 5
|
||||
workingDirectory: $(WORK_DIR)
|
||||
displayName: "Block Blob Stream Preparation"
|
||||
|
||||
# Code Coverage with e2e-tests for block blob with streaming on
|
||||
- script: |
|
||||
rm -rf $(MOUNT_DIR)/*
|
||||
rm -rf $(TEMP_DIR)/*
|
||||
./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/blobfuse2_stream.cov mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_STREAM_CFG) --foreground=true &
|
||||
sleep 10
|
||||
ps -aux | grep blobfuse2
|
||||
./blobfuse2 mount list
|
||||
for i in {10,50,100,200,500,1024}; do echo $i; done | parallel --will-cite -j 5 'diff $(WORK_DIR)/myfile_{} $(MOUNT_DIR)/myfile_{}'
|
||||
sudo fusermount -u $(MOUNT_DIR)
|
||||
sleep 5
|
||||
workingDirectory: $(WORK_DIR)
|
||||
displayName: "Block Blob Stream Coverage"
|
||||
|
||||
|
||||
# -------------------------------------------------------
|
||||
# Config Generation (Block Blob) for cli options
|
||||
- script: |
|
||||
|
|
@ -596,7 +514,7 @@ stages:
|
|||
sudo apt-get install python3-setuptools -y
|
||||
sudo apt install python3-pip -y
|
||||
sudo pip3 install mitmproxy
|
||||
mitmdump -w proxy_logs.txt &
|
||||
mitmdump -q -w proxy_logs.txt &
|
||||
displayName: 'Install & Start Proxy'
|
||||
|
||||
# Configure Proxy cert & env
|
||||
|
|
@ -674,7 +592,7 @@ stages:
|
|||
- script: |
|
||||
echo 'mode: count' > ./blobfuse2_coverage_raw.rpt
|
||||
tail -q -n +2 ./*.cov >> ./blobfuse2_coverage_raw.rpt
|
||||
cat ./blobfuse2_coverage_raw.rpt | grep -v mock_component | grep -v base_component | grep -v loopback | grep -v tools | grep -v "common/log" | grep -v "common/exectime" | grep -v "internal/stats_manager" | grep -v "main.go" | grep -v "component/azstorage/azauthmsi.go" | grep -v "component/azstorage/azauthcli.go" > ./blobfuse2_coverage.rpt
|
||||
cat ./blobfuse2_coverage_raw.rpt | grep -v mock_component | grep -v base_component | grep -v loopback | grep -v tools | grep -v "common/log" | grep -v "common/exectime" | grep -v "common/types.go" | grep -v "internal/stats_manager" | grep -v "main.go" | grep -v "component/azstorage/azauthmsi.go" | grep -v "component/azstorage/azauthspn.go" | grep -v "component/stream" | grep -v "component/azstorage/azauthcli.go" > ./blobfuse2_coverage.rpt
|
||||
go tool cover -func blobfuse2_coverage.rpt > ./blobfuse2_func_cover.rpt
|
||||
go tool cover -html=./blobfuse2_coverage.rpt -o ./blobfuse2_coverage.html
|
||||
go tool cover -html=./blobfuse2_ut.cov -o ./blobfuse2_ut.html
|
||||
|
|
@ -717,7 +635,6 @@ stages:
|
|||
matrix:
|
||||
Ubuntu-20:
|
||||
AgentName: 'blobfuse-ubuntu20'
|
||||
imageName: 'ubuntu-20.04'
|
||||
containerName: 'test-cnt-ubn-20'
|
||||
fuselib: 'libfuse3-dev'
|
||||
fuselib2: 'fuse3'
|
||||
|
|
@ -736,8 +653,6 @@ stages:
|
|||
value: '$(Pipeline.Workspace)/blobfuse2_tmp'
|
||||
- name: BLOBFUSE2_CFG
|
||||
value: '$(Pipeline.Workspace)/blobfuse2.yaml'
|
||||
- name: BLOBFUSE2_STREAM_CFG
|
||||
value: '$(Pipeline.Workspace)/blobfuse2_stream.yaml'
|
||||
- name: BLOBFUSE2_ADLS_CFG
|
||||
value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml'
|
||||
- name: GOPATH
|
||||
|
|
|
|||
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
|
|
@ -687,115 +687,6 @@ func (suite *authTestSuite) TestAdlskMsiResId() {
|
|||
}
|
||||
}
|
||||
|
||||
func (suite *authTestSuite) TestBlockInvalidSpn() {
|
||||
defer suite.cleanupTest()
|
||||
stgConfig := AzStorageConfig{
|
||||
container: storageTestConfigurationParameters.BlockContainer,
|
||||
authConfig: azAuthConfig{
|
||||
AuthMode: EAuthType.SPN(),
|
||||
AccountType: EAccountType.BLOCK(),
|
||||
AccountName: storageTestConfigurationParameters.BlockAccount,
|
||||
ClientID: storageTestConfigurationParameters.SpnClientId,
|
||||
TenantID: storageTestConfigurationParameters.SpnTenantId,
|
||||
ClientSecret: "",
|
||||
Endpoint: generateEndpoint(false, storageTestConfigurationParameters.BlockAccount, EAccountType.BLOCK()),
|
||||
},
|
||||
}
|
||||
assert := assert.New(suite.T())
|
||||
stg := NewAzStorageConnection(stgConfig)
|
||||
if stg == nil {
|
||||
assert.Fail("TestBlockInvalidSpn : Failed to create Storage object")
|
||||
}
|
||||
if err := stg.SetupPipeline(); err == nil {
|
||||
assert.Fail("TestBlockInvalidSpn : Setup pipeline even though spn is invalid")
|
||||
}
|
||||
}
|
||||
|
||||
func (suite *authTestSuite) TestBlockInvalidTokenPathSpn() {
|
||||
defer suite.cleanupTest()
|
||||
|
||||
_ = os.WriteFile("newtoken.txt", []byte("abcdef"), 0777)
|
||||
defer os.Remove("newtoken.txt")
|
||||
|
||||
stgConfig := AzStorageConfig{
|
||||
container: storageTestConfigurationParameters.BlockContainer,
|
||||
authConfig: azAuthConfig{
|
||||
AuthMode: EAuthType.SPN(),
|
||||
AccountType: EAccountType.BLOCK(),
|
||||
AccountName: storageTestConfigurationParameters.BlockAccount,
|
||||
ClientID: storageTestConfigurationParameters.SpnClientId,
|
||||
TenantID: storageTestConfigurationParameters.SpnTenantId,
|
||||
ClientSecret: "",
|
||||
Endpoint: generateEndpoint(false, storageTestConfigurationParameters.BlockAccount, EAccountType.BLOCK()),
|
||||
OAuthTokenFilePath: "newtoken.txt",
|
||||
},
|
||||
}
|
||||
assert := assert.New(suite.T())
|
||||
stg := NewAzStorageConnection(stgConfig)
|
||||
if stg == nil {
|
||||
assert.Fail("TestBlockInvalidSpn : Failed to create Storage object")
|
||||
}
|
||||
_ = stg.SetupPipeline()
|
||||
}
|
||||
|
||||
func (suite *authTestSuite) TestBlockSpn() {
|
||||
defer suite.cleanupTest()
|
||||
stgConfig := AzStorageConfig{
|
||||
container: storageTestConfigurationParameters.BlockContainer,
|
||||
authConfig: azAuthConfig{
|
||||
AuthMode: EAuthType.SPN(),
|
||||
AccountType: EAccountType.BLOCK(),
|
||||
AccountName: storageTestConfigurationParameters.BlockAccount,
|
||||
ClientID: storageTestConfigurationParameters.SpnClientId,
|
||||
TenantID: storageTestConfigurationParameters.SpnTenantId,
|
||||
ClientSecret: storageTestConfigurationParameters.SpnClientSecret,
|
||||
Endpoint: generateEndpoint(false, storageTestConfigurationParameters.BlockAccount, EAccountType.BLOCK()),
|
||||
},
|
||||
}
|
||||
suite.validateStorageTest("TestBlockSpn", stgConfig)
|
||||
}
|
||||
|
||||
func (suite *authTestSuite) TestAdlsInvalidSpn() {
|
||||
defer suite.cleanupTest()
|
||||
stgConfig := AzStorageConfig{
|
||||
container: storageTestConfigurationParameters.AdlsContainer,
|
||||
authConfig: azAuthConfig{
|
||||
AuthMode: EAuthType.SPN(),
|
||||
AccountType: EAccountType.ADLS(),
|
||||
AccountName: storageTestConfigurationParameters.AdlsAccount,
|
||||
ClientID: storageTestConfigurationParameters.SpnClientId,
|
||||
TenantID: storageTestConfigurationParameters.SpnTenantId,
|
||||
ClientSecret: "",
|
||||
Endpoint: generateEndpoint(false, storageTestConfigurationParameters.AdlsAccount, EAccountType.ADLS()),
|
||||
},
|
||||
}
|
||||
assert := assert.New(suite.T())
|
||||
stg := NewAzStorageConnection(stgConfig)
|
||||
if stg == nil {
|
||||
assert.Fail("TestAdlsInvalidSpn : Failed to create Storage object")
|
||||
}
|
||||
if err := stg.SetupPipeline(); err == nil {
|
||||
assert.Fail("TestAdlsInvalidSpn : Setup pipeline even though spn is invalid")
|
||||
}
|
||||
}
|
||||
|
||||
func (suite *authTestSuite) TestAdlsSpn() {
|
||||
defer suite.cleanupTest()
|
||||
stgConfig := AzStorageConfig{
|
||||
container: storageTestConfigurationParameters.AdlsContainer,
|
||||
authConfig: azAuthConfig{
|
||||
AuthMode: EAuthType.SPN(),
|
||||
AccountType: EAccountType.ADLS(),
|
||||
AccountName: storageTestConfigurationParameters.AdlsAccount,
|
||||
ClientID: storageTestConfigurationParameters.SpnClientId,
|
||||
TenantID: storageTestConfigurationParameters.SpnTenantId,
|
||||
ClientSecret: storageTestConfigurationParameters.SpnClientSecret,
|
||||
Endpoint: generateEndpoint(false, storageTestConfigurationParameters.AdlsAccount, EAccountType.ADLS()),
|
||||
},
|
||||
}
|
||||
suite.validateStorageTest("TestAdlsSpn", stgConfig)
|
||||
}
|
||||
|
||||
func (suite *authTestSuite) TestBlockAzCLI() {
|
||||
defer suite.cleanupTest()
|
||||
stgConfig := AzStorageConfig{
|
||||
|
|
|
|||
|
|
@ -294,16 +294,7 @@ func (c *FileCache) Configure(_ bool) error {
|
|||
}
|
||||
|
||||
cacheConfig := c.GetPolicyConfig(conf)
|
||||
|
||||
switch strings.ToLower(conf.Policy) {
|
||||
case "lru":
|
||||
c.policy = NewLRUPolicy(cacheConfig)
|
||||
case "lfu":
|
||||
c.policy = NewLFUPolicy(cacheConfig)
|
||||
default:
|
||||
log.Info("FileCache::Configure : Using default eviction policy")
|
||||
c.policy = NewLRUPolicy(cacheConfig)
|
||||
}
|
||||
c.policy = NewLRUPolicy(cacheConfig)
|
||||
|
||||
if c.policy == nil {
|
||||
log.Err("FileCache::Configure : failed to create cache eviction policy")
|
||||
|
|
|
|||
|
|
@ -161,7 +161,7 @@ func (suite *fileCacheTestSuite) TestEmpty() {
|
|||
func (suite *fileCacheTestSuite) TestConfig() {
|
||||
defer suite.cleanupTest()
|
||||
suite.cleanupTest() // teardown the default file cache generated
|
||||
policy := "lfu"
|
||||
policy := "lru"
|
||||
maxSizeMb := 1024
|
||||
cacheTimeout := 60
|
||||
maxDeletion := 10
|
||||
|
|
@ -178,10 +178,10 @@ func (suite *fileCacheTestSuite) TestConfig() {
|
|||
suite.assert.Equal(suite.fileCache.tmpPath, suite.cache_path)
|
||||
suite.assert.Equal(suite.fileCache.policy.Name(), policy)
|
||||
|
||||
suite.assert.EqualValues(suite.fileCache.policy.(*lfuPolicy).maxSizeMB, maxSizeMb)
|
||||
suite.assert.EqualValues(suite.fileCache.policy.(*lfuPolicy).maxEviction, maxDeletion)
|
||||
suite.assert.EqualValues(suite.fileCache.policy.(*lfuPolicy).highThreshold, highThreshold)
|
||||
suite.assert.EqualValues(suite.fileCache.policy.(*lfuPolicy).lowThreshold, lowThreshold)
|
||||
suite.assert.EqualValues(suite.fileCache.policy.(*lruPolicy).maxSizeMB, maxSizeMb)
|
||||
suite.assert.EqualValues(suite.fileCache.policy.(*lruPolicy).maxEviction, maxDeletion)
|
||||
suite.assert.EqualValues(suite.fileCache.policy.(*lruPolicy).highThreshold, highThreshold)
|
||||
suite.assert.EqualValues(suite.fileCache.policy.(*lruPolicy).lowThreshold, lowThreshold)
|
||||
|
||||
suite.assert.Equal(suite.fileCache.createEmptyFile, createEmptyFile)
|
||||
suite.assert.Equal(suite.fileCache.allowNonEmpty, allowNonEmptyTemp)
|
||||
|
|
@ -192,7 +192,7 @@ func (suite *fileCacheTestSuite) TestConfig() {
|
|||
func (suite *fileCacheTestSuite) TestConfigPolicyTimeout() {
|
||||
defer suite.cleanupTest()
|
||||
suite.cleanupTest() // teardown the default file cache generated
|
||||
policy := "lfu"
|
||||
policy := "lru"
|
||||
maxSizeMb := 1024
|
||||
cacheTimeout := 60
|
||||
maxDeletion := 10
|
||||
|
|
@ -209,11 +209,11 @@ func (suite *fileCacheTestSuite) TestConfigPolicyTimeout() {
|
|||
suite.assert.Equal(suite.fileCache.tmpPath, suite.cache_path)
|
||||
suite.assert.Equal(suite.fileCache.policy.Name(), policy)
|
||||
|
||||
suite.assert.EqualValues(suite.fileCache.policy.(*lfuPolicy).maxSizeMB, maxSizeMb)
|
||||
suite.assert.EqualValues(suite.fileCache.policy.(*lfuPolicy).maxEviction, maxDeletion)
|
||||
suite.assert.EqualValues(suite.fileCache.policy.(*lfuPolicy).highThreshold, highThreshold)
|
||||
suite.assert.EqualValues(suite.fileCache.policy.(*lfuPolicy).lowThreshold, lowThreshold)
|
||||
suite.assert.EqualValues(suite.fileCache.policy.(*lfuPolicy).cacheTimeout, cacheTimeout)
|
||||
suite.assert.EqualValues(suite.fileCache.policy.(*lruPolicy).maxSizeMB, maxSizeMb)
|
||||
suite.assert.EqualValues(suite.fileCache.policy.(*lruPolicy).maxEviction, maxDeletion)
|
||||
suite.assert.EqualValues(suite.fileCache.policy.(*lruPolicy).highThreshold, highThreshold)
|
||||
suite.assert.EqualValues(suite.fileCache.policy.(*lruPolicy).lowThreshold, lowThreshold)
|
||||
suite.assert.EqualValues(suite.fileCache.policy.(*lruPolicy).cacheTimeout, cacheTimeout)
|
||||
|
||||
suite.assert.Equal(suite.fileCache.createEmptyFile, createEmptyFile)
|
||||
suite.assert.Equal(suite.fileCache.allowNonEmpty, allowNonEmptyTemp)
|
||||
|
|
@ -224,7 +224,7 @@ func (suite *fileCacheTestSuite) TestConfigPolicyTimeout() {
|
|||
func (suite *fileCacheTestSuite) TestConfigPolicyDefaultTimeout() {
|
||||
defer suite.cleanupTest()
|
||||
suite.cleanupTest() // teardown the default file cache generated
|
||||
policy := "lfu"
|
||||
policy := "lru"
|
||||
maxSizeMb := 1024
|
||||
cacheTimeout := defaultFileCacheTimeout
|
||||
maxDeletion := 10
|
||||
|
|
@ -241,11 +241,11 @@ func (suite *fileCacheTestSuite) TestConfigPolicyDefaultTimeout() {
|
|||
suite.assert.Equal(suite.fileCache.tmpPath, suite.cache_path)
|
||||
suite.assert.Equal(suite.fileCache.policy.Name(), policy)
|
||||
|
||||
suite.assert.EqualValues(suite.fileCache.policy.(*lfuPolicy).maxSizeMB, maxSizeMb)
|
||||
suite.assert.EqualValues(suite.fileCache.policy.(*lfuPolicy).maxEviction, maxDeletion)
|
||||
suite.assert.EqualValues(suite.fileCache.policy.(*lfuPolicy).highThreshold, highThreshold)
|
||||
suite.assert.EqualValues(suite.fileCache.policy.(*lfuPolicy).lowThreshold, lowThreshold)
|
||||
suite.assert.EqualValues(suite.fileCache.policy.(*lfuPolicy).cacheTimeout, cacheTimeout)
|
||||
suite.assert.EqualValues(suite.fileCache.policy.(*lruPolicy).maxSizeMB, maxSizeMb)
|
||||
suite.assert.EqualValues(suite.fileCache.policy.(*lruPolicy).maxEviction, maxDeletion)
|
||||
suite.assert.EqualValues(suite.fileCache.policy.(*lruPolicy).highThreshold, highThreshold)
|
||||
suite.assert.EqualValues(suite.fileCache.policy.(*lruPolicy).lowThreshold, lowThreshold)
|
||||
suite.assert.EqualValues(suite.fileCache.policy.(*lruPolicy).cacheTimeout, cacheTimeout)
|
||||
|
||||
suite.assert.Equal(suite.fileCache.createEmptyFile, createEmptyFile)
|
||||
suite.assert.Equal(suite.fileCache.allowNonEmpty, allowNonEmptyTemp)
|
||||
|
|
@ -256,7 +256,7 @@ func (suite *fileCacheTestSuite) TestConfigPolicyDefaultTimeout() {
|
|||
func (suite *fileCacheTestSuite) TestConfigZero() {
|
||||
defer suite.cleanupTest()
|
||||
suite.cleanupTest() // teardown the default file cache generated
|
||||
policy := "lfu"
|
||||
policy := "lru"
|
||||
maxSizeMb := 1024
|
||||
cacheTimeout := 0
|
||||
maxDeletion := 10
|
||||
|
|
@ -273,10 +273,10 @@ func (suite *fileCacheTestSuite) TestConfigZero() {
|
|||
suite.assert.Equal(suite.fileCache.tmpPath, suite.cache_path)
|
||||
suite.assert.Equal(suite.fileCache.policy.Name(), policy)
|
||||
|
||||
suite.assert.EqualValues(suite.fileCache.policy.(*lfuPolicy).maxSizeMB, maxSizeMb)
|
||||
suite.assert.EqualValues(suite.fileCache.policy.(*lfuPolicy).maxEviction, maxDeletion)
|
||||
suite.assert.EqualValues(suite.fileCache.policy.(*lfuPolicy).highThreshold, highThreshold)
|
||||
suite.assert.EqualValues(suite.fileCache.policy.(*lfuPolicy).lowThreshold, lowThreshold)
|
||||
suite.assert.EqualValues(suite.fileCache.policy.(*lruPolicy).maxSizeMB, maxSizeMb)
|
||||
suite.assert.EqualValues(suite.fileCache.policy.(*lruPolicy).maxEviction, maxDeletion)
|
||||
suite.assert.EqualValues(suite.fileCache.policy.(*lruPolicy).highThreshold, highThreshold)
|
||||
suite.assert.EqualValues(suite.fileCache.policy.(*lruPolicy).lowThreshold, lowThreshold)
|
||||
|
||||
suite.assert.Equal(suite.fileCache.createEmptyFile, createEmptyFile)
|
||||
suite.assert.Equal(suite.fileCache.allowNonEmpty, allowNonEmptyTemp)
|
||||
|
|
|
|||
|
|
@ -1,483 +0,0 @@
|
|||
/*
|
||||
_____ _____ _____ ____ ______ _____ ------
|
||||
| | | | | | | | | | | | |
|
||||
| | | | | | | | | | | | |
|
||||
| --- | | | | |-----| |---- | | |-----| |----- ------
|
||||
| | | | | | | | | | | | |
|
||||
| ____| |_____ | ____| | ____| | |_____| _____| |_____ |_____
|
||||
|
||||
|
||||
Licensed under the MIT License <http://opensource.org/licenses/MIT>.
|
||||
|
||||
Copyright © 2020-2024 Microsoft Corporation. All rights reserved.
|
||||
Author : <blobfusedev@microsoft.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE
|
||||
*/
|
||||
|
||||
package file_cache
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-storage-fuse/v2/common/log"
|
||||
)
|
||||
|
||||
type lfuPolicy struct {
|
||||
sync.Mutex
|
||||
cachePolicyConfig
|
||||
list *lfuList
|
||||
removeFiles chan string
|
||||
closeChan chan int
|
||||
}
|
||||
|
||||
var _ cachePolicy = &lfuPolicy{}
|
||||
|
||||
func (l *lfuPolicy) StartPolicy() error {
|
||||
log.Trace("lfuPolicy::StartPolicy")
|
||||
|
||||
go l.clearCache()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *lfuPolicy) ShutdownPolicy() error {
|
||||
log.Trace("lfuPolicy::ShutdownPolicy")
|
||||
|
||||
l.closeChan <- 1
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *lfuPolicy) UpdateConfig(config cachePolicyConfig) error {
|
||||
log.Trace("lfuPolicy::UpdateConfig")
|
||||
|
||||
l.maxSizeMB = config.maxSizeMB
|
||||
l.highThreshold = config.highThreshold
|
||||
l.lowThreshold = config.lowThreshold
|
||||
l.maxEviction = config.maxEviction
|
||||
|
||||
l.list.maxSizeMB = config.maxSizeMB
|
||||
l.list.upperThresh = config.highThreshold
|
||||
l.list.lowerThresh = config.lowThreshold
|
||||
l.list.cacheTimeout = config.cacheTimeout
|
||||
|
||||
l.policyTrace = config.policyTrace
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *lfuPolicy) CacheValid(name string) {
|
||||
log.Trace("lfuPolicy::CacheValid : %s", name)
|
||||
|
||||
l.list.Lock()
|
||||
defer l.list.Unlock()
|
||||
|
||||
l.list.put(name)
|
||||
}
|
||||
|
||||
func (l *lfuPolicy) CacheInvalidate(name string) {
|
||||
log.Trace("lfuPolicy::CacheInvalidate : %s", name)
|
||||
|
||||
if l.cacheTimeout == 0 {
|
||||
l.CachePurge(name)
|
||||
}
|
||||
}
|
||||
|
||||
func (l *lfuPolicy) CachePurge(name string) {
|
||||
log.Trace("lfuPolicy::CachePurge : %s", name)
|
||||
|
||||
l.list.Lock()
|
||||
defer l.list.Unlock()
|
||||
|
||||
l.list.delete(name)
|
||||
l.removeFiles <- name
|
||||
}
|
||||
|
||||
func (l *lfuPolicy) IsCached(name string) bool {
|
||||
log.Trace("lfuPolicy::IsCached : %s", name)
|
||||
|
||||
l.list.Lock()
|
||||
defer l.list.Unlock()
|
||||
|
||||
val := l.list.get(name)
|
||||
if val != nil {
|
||||
log.Debug("lfuPolicy::IsCached : %s found", name)
|
||||
return true
|
||||
} else {
|
||||
log.Debug("lfuPolicy::IsCached : %s not found", name)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (l *lfuPolicy) Name() string {
|
||||
return "lfu"
|
||||
}
|
||||
|
||||
func (l *lfuPolicy) clearItemFromCache(path string) {
|
||||
azPath := strings.TrimPrefix(path, l.tmpPath)
|
||||
if azPath[0] == '/' {
|
||||
azPath = azPath[1:]
|
||||
}
|
||||
|
||||
flock := l.fileLocks.Get(azPath)
|
||||
if l.fileLocks.Locked(azPath) {
|
||||
log.Warn("lfuPolicy::DeleteItem : File in under download %s", azPath)
|
||||
l.CacheValid(path)
|
||||
return
|
||||
}
|
||||
|
||||
flock.Lock()
|
||||
defer flock.Unlock()
|
||||
|
||||
// Check if there are any open handles to this file or not
|
||||
if flock.Count() > 0 {
|
||||
log.Err("lfuPolicy::clearItemFromCache : File in use %s", path)
|
||||
l.CacheValid(path)
|
||||
return
|
||||
}
|
||||
|
||||
// There are no open handles for this file so its safe to remove this
|
||||
err := deleteFile(path)
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
log.Err("lfuPolicy::DeleteItem : failed to delete local file %s [%s]", path, err.Error())
|
||||
}
|
||||
|
||||
// File was deleted so try clearing its parent directory
|
||||
// TODO: Delete directories up the path recursively that are "safe to delete". Ensure there is no race between this code and code that creates directories (like OpenFile)
|
||||
// This might require something like hierarchical locking.
|
||||
}
|
||||
|
||||
func (l *lfuPolicy) clearCache() {
|
||||
log.Trace("lfuPolicy::clearCache")
|
||||
|
||||
for {
|
||||
select {
|
||||
|
||||
case path := <-l.removeFiles:
|
||||
l.clearItemFromCache(path)
|
||||
|
||||
case <-l.closeChan:
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func NewLFUPolicy(cfg cachePolicyConfig) cachePolicy {
|
||||
pol := &lfuPolicy{
|
||||
cachePolicyConfig: cfg,
|
||||
removeFiles: make(chan string, 10),
|
||||
closeChan: make(chan int, 10),
|
||||
}
|
||||
pol.list = newLFUList(cfg.maxSizeMB, cfg.lowThreshold, cfg.highThreshold, pol.removeFiles, cfg.tmpPath, cfg.cacheTimeout)
|
||||
return pol
|
||||
}
|
||||
|
||||
//Double DoublyLinkedList Implementation for O(1) lfu
|
||||
|
||||
type dataNode struct {
|
||||
key string
|
||||
frequency uint64
|
||||
next *dataNode
|
||||
prev *dataNode
|
||||
timer *time.Timer
|
||||
}
|
||||
|
||||
func newDataNode(key string) *dataNode {
|
||||
return &dataNode{
|
||||
key: key,
|
||||
frequency: 1,
|
||||
}
|
||||
}
|
||||
|
||||
type dataNodeLinkedList struct {
|
||||
size uint64
|
||||
first *dataNode
|
||||
last *dataNode
|
||||
}
|
||||
|
||||
func (dl *dataNodeLinkedList) pop() *dataNode {
|
||||
if dl.size == 0 {
|
||||
return nil
|
||||
}
|
||||
return dl.remove(dl.first)
|
||||
}
|
||||
|
||||
func (dl *dataNodeLinkedList) remove(node *dataNode) *dataNode {
|
||||
if dl.size == 0 {
|
||||
return nil
|
||||
}
|
||||
if dl.first == dl.last {
|
||||
dl.first = nil
|
||||
dl.last = nil
|
||||
} else if dl.first == node {
|
||||
temp := dl.first
|
||||
dl.first = temp.next
|
||||
temp.next = nil
|
||||
dl.first.prev = nil
|
||||
} else if dl.last == node {
|
||||
temp := dl.last
|
||||
dl.last = temp.prev
|
||||
temp.prev = nil
|
||||
dl.last.next = nil
|
||||
} else {
|
||||
nextNode := node.next
|
||||
prevNode := node.prev
|
||||
prevNode.next = nextNode
|
||||
nextNode.prev = prevNode
|
||||
node.next = nil
|
||||
node.prev = nil
|
||||
}
|
||||
dl.size--
|
||||
return node
|
||||
}
|
||||
|
||||
func (dl *dataNodeLinkedList) push(node *dataNode) {
|
||||
if dl.first == nil {
|
||||
dl.first = node
|
||||
dl.last = node
|
||||
} else {
|
||||
temp := dl.last
|
||||
temp.next = node
|
||||
node.prev = temp
|
||||
dl.last = node
|
||||
}
|
||||
dl.size++
|
||||
}
|
||||
|
||||
func newDataNodeLinkedList() *dataNodeLinkedList {
|
||||
return &dataNodeLinkedList{}
|
||||
}
|
||||
|
||||
type frequencyNode struct {
|
||||
list *dataNodeLinkedList
|
||||
next *frequencyNode
|
||||
prev *frequencyNode
|
||||
frequency uint64
|
||||
}
|
||||
|
||||
func (fn *frequencyNode) pop() *dataNode {
|
||||
return fn.list.pop()
|
||||
}
|
||||
|
||||
func (fn *frequencyNode) remove(dn *dataNode) *dataNode {
|
||||
return fn.list.remove(dn)
|
||||
}
|
||||
|
||||
func (fn *frequencyNode) push(dn *dataNode) {
|
||||
fn.list.push(dn)
|
||||
}
|
||||
|
||||
func newFrequencyNode(freq uint64) *frequencyNode {
|
||||
return &frequencyNode{
|
||||
list: newDataNodeLinkedList(),
|
||||
frequency: freq,
|
||||
}
|
||||
}
|
||||
|
||||
type lfuList struct {
|
||||
sync.Mutex
|
||||
first *frequencyNode
|
||||
last *frequencyNode
|
||||
dataNodeMap map[string]*dataNode
|
||||
freqNodeMap map[uint64]*frequencyNode
|
||||
size uint64
|
||||
maxSizeMB float64
|
||||
lowerThresh float64
|
||||
upperThresh float64
|
||||
deleteFiles chan string
|
||||
cachePath string
|
||||
cacheAge uint64
|
||||
cacheTimeout uint32
|
||||
}
|
||||
|
||||
func (list *lfuList) deleteFrequency(freq uint64) {
|
||||
freqNode := list.freqNodeMap[freq]
|
||||
if list.first == list.last {
|
||||
list.first = nil
|
||||
list.last = nil
|
||||
} else if list.first == freqNode {
|
||||
list.first = list.first.next
|
||||
list.first.prev = nil
|
||||
freqNode.next = nil
|
||||
} else if list.last == freqNode {
|
||||
list.last = list.last.prev
|
||||
list.last.next = nil
|
||||
freqNode.prev = nil
|
||||
} else {
|
||||
nextNode := freqNode.next
|
||||
prevNode := freqNode.prev
|
||||
nextNode.prev = prevNode
|
||||
prevNode.next = nextNode
|
||||
freqNode.next = nil
|
||||
freqNode.prev = nil
|
||||
}
|
||||
list.size--
|
||||
delete(list.freqNodeMap, freq)
|
||||
}
|
||||
|
||||
func (list *lfuList) addFrequency(freq uint64, freqNode *frequencyNode, prevFreqNode *frequencyNode) {
|
||||
|
||||
if list.first == nil && list.last == nil {
|
||||
list.first = freqNode
|
||||
list.last = freqNode
|
||||
|
||||
list.freqNodeMap[freq] = freqNode
|
||||
list.size++
|
||||
return
|
||||
}
|
||||
|
||||
if prevFreqNode == nil {
|
||||
prevFreqNode = list.first
|
||||
}
|
||||
|
||||
for prevFreqNode.next != nil && freq > prevFreqNode.next.frequency {
|
||||
prevFreqNode = prevFreqNode.next
|
||||
}
|
||||
|
||||
if prevFreqNode == nil {
|
||||
freqNode.next = list.first
|
||||
list.first.prev = freqNode
|
||||
list.first = freqNode
|
||||
} else if prevFreqNode == list.last {
|
||||
prevFreqNode.next = freqNode
|
||||
freqNode.prev = prevFreqNode
|
||||
list.last = freqNode
|
||||
} else {
|
||||
nextNode := prevFreqNode.next
|
||||
freqNode.next = nextNode
|
||||
nextNode.prev = freqNode
|
||||
prevFreqNode.next = freqNode
|
||||
freqNode.prev = prevFreqNode
|
||||
}
|
||||
list.freqNodeMap[freq] = freqNode
|
||||
list.size++
|
||||
}
|
||||
|
||||
func (list *lfuList) promote(dataNode *dataNode) {
|
||||
prevFreqNode := list.freqNodeMap[dataNode.frequency]
|
||||
prevFreqNode.remove(dataNode)
|
||||
dataNode.frequency += 1 + list.cacheAge
|
||||
if newFreqNode, ok := list.freqNodeMap[dataNode.frequency]; ok {
|
||||
newFreqNode.push(dataNode)
|
||||
} else {
|
||||
newFreqNode := newFrequencyNode(dataNode.frequency)
|
||||
list.addFrequency(dataNode.frequency, newFreqNode, prevFreqNode)
|
||||
list.freqNodeMap[dataNode.frequency] = newFreqNode
|
||||
newFreqNode.push(dataNode)
|
||||
}
|
||||
|
||||
if prevFreqNode.list.size == 0 {
|
||||
list.deleteFrequency(prevFreqNode.frequency)
|
||||
list.size--
|
||||
}
|
||||
}
|
||||
|
||||
func (list *lfuList) get(key string) *dataNode {
|
||||
if node, ok := list.dataNodeMap[key]; ok {
|
||||
if list.cacheTimeout > 0 {
|
||||
node.timer.Stop()
|
||||
}
|
||||
list.promote(node)
|
||||
list.setTimerIfValid(node)
|
||||
return node
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// Requires Lock()
|
||||
func (list *lfuList) put(key string) {
|
||||
if node, ok := list.dataNodeMap[key]; ok {
|
||||
if list.cacheTimeout > 0 {
|
||||
node.timer.Stop()
|
||||
}
|
||||
list.promote(node)
|
||||
list.setTimerIfValid(node)
|
||||
} else {
|
||||
if usage := getUsagePercentage(list.cachePath, list.maxSizeMB); usage > list.upperThresh {
|
||||
for usage > list.lowerThresh && list.first != nil {
|
||||
toDeletePath := list.first.list.first.key
|
||||
list.first.pop()
|
||||
delete(list.dataNodeMap, toDeletePath)
|
||||
if list.first.list.size == 0 {
|
||||
list.deleteFrequency(list.first.frequency)
|
||||
list.size--
|
||||
usage = getUsagePercentage(list.cachePath, list.maxSizeMB)
|
||||
}
|
||||
list.deleteFiles <- toDeletePath
|
||||
}
|
||||
}
|
||||
newNode := newDataNode(key)
|
||||
list.dataNodeMap[key] = newNode
|
||||
if freqNode, ok := list.freqNodeMap[newNode.frequency]; ok {
|
||||
freqNode.push(newNode)
|
||||
} else {
|
||||
freqNode := newFrequencyNode(newNode.frequency)
|
||||
list.freqNodeMap[newNode.frequency] = freqNode
|
||||
freqNode.push(newNode)
|
||||
list.addFrequency(newNode.frequency, freqNode, nil)
|
||||
}
|
||||
list.setTimerIfValid(newNode)
|
||||
}
|
||||
}
|
||||
|
||||
// Requires Lock()
|
||||
func (list *lfuList) delete(key string) {
|
||||
if node, ok := list.dataNodeMap[key]; ok {
|
||||
if list.cacheTimeout > 0 {
|
||||
node.timer.Stop()
|
||||
}
|
||||
freqNode := list.freqNodeMap[node.frequency]
|
||||
freqNode.remove(node)
|
||||
delete(list.dataNodeMap, key)
|
||||
if freqNode.list.size == 0 {
|
||||
list.deleteFrequency(node.frequency)
|
||||
list.size--
|
||||
}
|
||||
list.deleteFiles <- node.key
|
||||
list.cacheAge = node.frequency
|
||||
}
|
||||
}
|
||||
|
||||
func (list *lfuList) setTimerIfValid(node *dataNode) {
|
||||
if list.cacheTimeout > 0 {
|
||||
timer := time.AfterFunc(time.Duration(list.cacheTimeout)*time.Second, func() {
|
||||
list.Lock()
|
||||
list.delete(node.key)
|
||||
list.Unlock()
|
||||
})
|
||||
node.timer = timer
|
||||
}
|
||||
}
|
||||
|
||||
func newLFUList(maxSizMB float64, lowerThresh float64, upperThresh float64, deleteChan chan string, cachePath string, cacheTimeout uint32) *lfuList {
|
||||
return &lfuList{
|
||||
dataNodeMap: make(map[string]*dataNode),
|
||||
freqNodeMap: make(map[uint64]*frequencyNode),
|
||||
size: 0,
|
||||
maxSizeMB: maxSizMB,
|
||||
lowerThresh: lowerThresh,
|
||||
upperThresh: upperThresh,
|
||||
deleteFiles: deleteChan,
|
||||
cachePath: cachePath,
|
||||
cacheTimeout: cacheTimeout,
|
||||
}
|
||||
}
|
||||
|
|
@ -1,286 +0,0 @@
|
|||
/*
|
||||
_____ _____ _____ ____ ______ _____ ------
|
||||
| | | | | | | | | | | | |
|
||||
| | | | | | | | | | | | |
|
||||
| --- | | | | |-----| |---- | | |-----| |----- ------
|
||||
| | | | | | | | | | | | |
|
||||
| ____| |_____ | ____| | ____| | |_____| _____| |_____ |_____
|
||||
|
||||
|
||||
Licensed under the MIT License <http://opensource.org/licenses/MIT>.
|
||||
|
||||
Copyright © 2020-2024 Microsoft Corporation. All rights reserved.
|
||||
Author : <blobfusedev@microsoft.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE
|
||||
*/
|
||||
|
||||
package file_cache
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-storage-fuse/v2/common"
|
||||
"github.com/Azure/azure-storage-fuse/v2/common/log"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
|
||||
type lfuPolicyTestSuite struct {
|
||||
suite.Suite
|
||||
assert *assert.Assertions
|
||||
policy *lfuPolicy
|
||||
}
|
||||
|
||||
var cache_path = filepath.Join(home_dir, "file_cache")
|
||||
|
||||
func (suite *lfuPolicyTestSuite) SetupTest() {
|
||||
err := log.SetDefaultLogger("silent", common.LogConfig{Level: common.ELogLevel.LOG_DEBUG()})
|
||||
if err != nil {
|
||||
panic("Unable to set silent logger as default.")
|
||||
}
|
||||
suite.assert = assert.New(suite.T())
|
||||
|
||||
os.Mkdir(cache_path, fs.FileMode(0777))
|
||||
|
||||
config := cachePolicyConfig{
|
||||
tmpPath: cache_path,
|
||||
cacheTimeout: 0,
|
||||
maxEviction: defaultMaxEviction,
|
||||
maxSizeMB: 0,
|
||||
highThreshold: defaultMaxThreshold,
|
||||
lowThreshold: defaultMinThreshold,
|
||||
fileLocks: &common.LockMap{},
|
||||
}
|
||||
|
||||
suite.setupTestHelper(config)
|
||||
}
|
||||
|
||||
func (suite *lfuPolicyTestSuite) setupTestHelper(config cachePolicyConfig) {
|
||||
suite.policy = NewLFUPolicy(config).(*lfuPolicy)
|
||||
suite.policy.StartPolicy()
|
||||
}
|
||||
|
||||
func (suite *lfuPolicyTestSuite) cleanupTest() {
|
||||
suite.policy.ShutdownPolicy()
|
||||
|
||||
os.RemoveAll(cache_path)
|
||||
}
|
||||
|
||||
func (suite *lfuPolicyTestSuite) TestDefault() {
|
||||
defer suite.cleanupTest()
|
||||
suite.assert.EqualValues("lfu", suite.policy.Name())
|
||||
suite.assert.EqualValues(0, suite.policy.cacheTimeout) // cacheTimeout does not change
|
||||
suite.assert.EqualValues(defaultMaxEviction, suite.policy.maxEviction)
|
||||
suite.assert.EqualValues(0, suite.policy.maxSizeMB)
|
||||
suite.assert.EqualValues(defaultMaxThreshold, suite.policy.highThreshold)
|
||||
suite.assert.EqualValues(defaultMinThreshold, suite.policy.lowThreshold)
|
||||
}
|
||||
|
||||
func (suite *lfuPolicyTestSuite) TestUpdateConfig() {
|
||||
defer suite.cleanupTest()
|
||||
config := cachePolicyConfig{
|
||||
tmpPath: cache_path,
|
||||
cacheTimeout: 120,
|
||||
maxEviction: 100,
|
||||
maxSizeMB: 10,
|
||||
highThreshold: 70,
|
||||
lowThreshold: 20,
|
||||
fileLocks: &common.LockMap{},
|
||||
}
|
||||
suite.policy.UpdateConfig(config)
|
||||
|
||||
suite.assert.NotEqualValues(120, suite.policy.cacheTimeout) // cacheTimeout does not change
|
||||
suite.assert.EqualValues(0, suite.policy.cacheTimeout) // cacheTimeout does not change
|
||||
suite.assert.EqualValues(100, suite.policy.maxEviction)
|
||||
suite.assert.EqualValues(10, suite.policy.maxSizeMB)
|
||||
suite.assert.EqualValues(70, suite.policy.highThreshold)
|
||||
suite.assert.EqualValues(20, suite.policy.lowThreshold)
|
||||
}
|
||||
|
||||
func (suite *lfuPolicyTestSuite) TestCacheValidNew() {
|
||||
defer suite.cleanupTest()
|
||||
suite.policy.CacheValid("temp")
|
||||
|
||||
node := suite.policy.list.get("temp")
|
||||
suite.assert.NotNil(node)
|
||||
suite.assert.EqualValues("temp", node.key)
|
||||
suite.assert.EqualValues(2, node.frequency) // the get will promote the node
|
||||
}
|
||||
|
||||
func (suite *lfuPolicyTestSuite) TestClearItemFromCache() {
|
||||
defer suite.cleanupTest()
|
||||
f, _ := os.Create(cache_path + "/test")
|
||||
suite.policy.clearItemFromCache(f.Name())
|
||||
_, attr := os.Stat(f.Name())
|
||||
suite.assert.NotEqual(nil, attr.Error())
|
||||
}
|
||||
|
||||
func (suite *lfuPolicyTestSuite) TestCacheValidExisting() {
|
||||
defer suite.cleanupTest()
|
||||
suite.policy.CacheValid("temp")
|
||||
|
||||
suite.policy.CacheValid("temp")
|
||||
node := suite.policy.list.get("temp")
|
||||
suite.assert.NotNil(node)
|
||||
suite.assert.EqualValues("temp", node.key)
|
||||
suite.assert.EqualValues(3, node.frequency) // the get will promote the node
|
||||
}
|
||||
|
||||
func (suite *lfuPolicyTestSuite) TestCacheInvalidate() {
|
||||
defer suite.cleanupTest()
|
||||
suite.policy.CacheValid("temp")
|
||||
suite.policy.CacheInvalidate("temp") // this is equivalent to purge since timeout=0
|
||||
|
||||
node := suite.policy.list.get("temp")
|
||||
suite.assert.Nil(node)
|
||||
}
|
||||
|
||||
func (suite *lfuPolicyTestSuite) TestCacheInvalidateTimeout() {
|
||||
defer suite.cleanupTest()
|
||||
suite.cleanupTest()
|
||||
|
||||
config := cachePolicyConfig{
|
||||
tmpPath: cache_path,
|
||||
cacheTimeout: 1,
|
||||
maxEviction: defaultMaxEviction,
|
||||
maxSizeMB: 0,
|
||||
highThreshold: defaultMaxThreshold,
|
||||
lowThreshold: defaultMinThreshold,
|
||||
fileLocks: &common.LockMap{},
|
||||
}
|
||||
|
||||
suite.setupTestHelper(config)
|
||||
|
||||
suite.policy.CacheValid("temp")
|
||||
suite.policy.CacheInvalidate("temp")
|
||||
|
||||
node := suite.policy.list.get("temp")
|
||||
suite.assert.NotNil(node)
|
||||
suite.assert.EqualValues("temp", node.key)
|
||||
suite.assert.EqualValues(2, node.frequency) // the get will promote the node
|
||||
}
|
||||
|
||||
func (suite *lfuPolicyTestSuite) TestCachePurge() {
|
||||
defer suite.cleanupTest()
|
||||
suite.policy.CacheValid("temp")
|
||||
suite.policy.CachePurge("temp")
|
||||
|
||||
node := suite.policy.list.get("temp")
|
||||
suite.assert.Nil(node)
|
||||
}
|
||||
|
||||
func (suite *lfuPolicyTestSuite) TestIsCached() {
|
||||
defer suite.cleanupTest()
|
||||
suite.policy.CacheValid("temp")
|
||||
|
||||
suite.assert.True(suite.policy.IsCached("temp"))
|
||||
}
|
||||
|
||||
func (suite *lfuPolicyTestSuite) TestIsCachedFalse() {
|
||||
defer suite.cleanupTest()
|
||||
suite.assert.False(suite.policy.IsCached("temp"))
|
||||
}
|
||||
|
||||
func (suite *lfuPolicyTestSuite) TestTimeout() {
|
||||
defer suite.cleanupTest()
|
||||
suite.cleanupTest()
|
||||
|
||||
config := cachePolicyConfig{
|
||||
tmpPath: cache_path,
|
||||
cacheTimeout: 1,
|
||||
maxEviction: defaultMaxEviction,
|
||||
maxSizeMB: 0,
|
||||
highThreshold: defaultMaxThreshold,
|
||||
lowThreshold: defaultMinThreshold,
|
||||
fileLocks: &common.LockMap{},
|
||||
}
|
||||
|
||||
suite.setupTestHelper(config)
|
||||
|
||||
suite.policy.CacheValid("temp")
|
||||
|
||||
time.Sleep(5 * time.Second) // Wait for time > cacheTimeout, the file should no longer be cached
|
||||
|
||||
suite.assert.False(suite.policy.IsCached("temp"))
|
||||
}
|
||||
|
||||
func (suite *lfuPolicyTestSuite) TestMaxEvictionDefault() {
|
||||
defer suite.cleanupTest()
|
||||
suite.cleanupTest()
|
||||
|
||||
config := cachePolicyConfig{
|
||||
tmpPath: cache_path,
|
||||
cacheTimeout: 1,
|
||||
maxEviction: defaultMaxEviction,
|
||||
maxSizeMB: 0,
|
||||
highThreshold: defaultMaxThreshold,
|
||||
lowThreshold: defaultMinThreshold,
|
||||
fileLocks: &common.LockMap{},
|
||||
}
|
||||
|
||||
suite.setupTestHelper(config)
|
||||
|
||||
for i := 1; i < 5000; i++ {
|
||||
suite.policy.CacheValid("temp" + fmt.Sprint(i))
|
||||
}
|
||||
|
||||
time.Sleep(5 * time.Second) // Wait for time > cacheTimeout, the file should no longer be cached
|
||||
|
||||
for i := 1; i < 5000; i++ {
|
||||
suite.assert.False(suite.policy.IsCached("temp" + fmt.Sprint(i)))
|
||||
}
|
||||
}
|
||||
|
||||
func (suite *lfuPolicyTestSuite) TestMaxEviction() {
|
||||
defer suite.cleanupTest()
|
||||
suite.cleanupTest()
|
||||
|
||||
config := cachePolicyConfig{
|
||||
tmpPath: cache_path,
|
||||
cacheTimeout: 1,
|
||||
maxEviction: 5,
|
||||
maxSizeMB: 0,
|
||||
highThreshold: defaultMaxThreshold,
|
||||
lowThreshold: defaultMinThreshold,
|
||||
fileLocks: &common.LockMap{},
|
||||
}
|
||||
|
||||
suite.setupTestHelper(config)
|
||||
|
||||
for i := 1; i < 5; i++ {
|
||||
suite.policy.CacheValid("temp" + fmt.Sprint(i))
|
||||
}
|
||||
|
||||
time.Sleep(5 * time.Second) // Wait for time > cacheTimeout, the file should no longer be cached
|
||||
|
||||
for i := 1; i < 5; i++ {
|
||||
suite.assert.False(suite.policy.IsCached("temp" + fmt.Sprint(i)))
|
||||
}
|
||||
}
|
||||
|
||||
func TestLFUPolicyTestSuite(t *testing.T) {
|
||||
suite.Run(t, new(lfuPolicyTestSuite))
|
||||
}
|
||||
|
|
@ -37,6 +37,7 @@ import (
|
|||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
|
|
@ -52,6 +53,8 @@ type lruPolicyTestSuite struct {
|
|||
policy *lruPolicy
|
||||
}
|
||||
|
||||
var cache_path = filepath.Join(home_dir, "file_cache")
|
||||
|
||||
func (suite *lruPolicyTestSuite) SetupTest() {
|
||||
// err := log.SetDefaultLogger("silent", common.LogConfig{Level: common.ELogLevel.LOG_DEBUG()})
|
||||
// if err != nil {
|
||||
|
|
|
|||
14
go.mod
14
go.mod
|
|
@ -3,10 +3,10 @@ module github.com/Azure/azure-storage-fuse/v2
|
|||
go 1.20
|
||||
|
||||
require (
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.1.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.1.3
|
||||
github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda
|
||||
github.com/fsnotify/fsnotify v1.7.0
|
||||
github.com/golang/mock v1.6.0
|
||||
|
|
@ -15,9 +15,9 @@ require (
|
|||
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
|
||||
github.com/radovskyb/watcher v1.0.7
|
||||
github.com/sevlyar/go-daemon v0.1.6
|
||||
github.com/spf13/cobra v1.8.0
|
||||
github.com/spf13/cobra v1.8.1
|
||||
github.com/spf13/pflag v1.0.5
|
||||
github.com/spf13/viper v1.18.2
|
||||
github.com/spf13/viper v1.19.0
|
||||
github.com/stretchr/testify v1.9.0
|
||||
github.com/vibhansa-msft/tlru v0.0.0-20240410102558-9e708419e21f
|
||||
go.uber.org/atomic v1.11.0
|
||||
|
|
@ -27,7 +27,7 @@ require (
|
|||
)
|
||||
|
||||
require (
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.0 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
|
|
@ -42,7 +42,7 @@ require (
|
|||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||
github.com/sagikazarmark/locafero v0.4.0 // indirect
|
||||
github.com/sagikazarmark/locafero v0.6.0 // indirect
|
||||
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
|
||||
github.com/sourcegraph/conc v0.3.0 // indirect
|
||||
github.com/spf13/afero v1.11.0 // indirect
|
||||
|
|
@ -50,7 +50,7 @@ require (
|
|||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
golang.org/x/crypto v0.24.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect
|
||||
golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect
|
||||
golang.org/x/net v0.26.0 // indirect
|
||||
golang.org/x/sys v0.21.0 // indirect
|
||||
golang.org/x/text v0.16.0 // indirect
|
||||
|
|
|
|||
24
go.sum
24
go.sum
|
|
@ -1,14 +1,14 @@
|
|||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 h1:E+OJmp2tPvt1W+amx48v1eqbjDYsgN+RzP4q16yV5eM=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1/go.mod h1:a6xsAQUZg+VsS3TJ05SRp524Hs4pZ/AeFSr5ENf0Yjo=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0 h1:1nGuui+4POelzDwI7RG56yfQJHCnKvwfMoU7VsEp+Zg=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0/go.mod h1:99EvauvlcJ1U06amZiksfYz/3aFGyIhWGHVyiZXtBAI=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0 h1:U2rTu3Ef+7w9FHKIAXM6ZyqF3UOWJZ12zIm8zECAFfg=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 h1:jBQA3cKT4L2rWMpgE7Yt3Hwh2aUj8KXjIGLxjHeYNNo=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0/go.mod h1:4OG6tQ9EOP/MT0NMjDlRzWoVFxfu9rN9B2X+tlSVktg=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.0 h1:H+U3Gk9zY56G3u872L82bk4thcsy2Gghb9ExT4Zvm1o=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.0/go.mod h1:mgrmMSgaLp9hmax62XQTd0N4aAqSE5E0DulSpVYK7vc=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2 h1:YUUxeiOWgdAQE3pXt2H7QXzZs0q8UBjgRbl56qo8GYM=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2/go.mod h1:dmXQgZuiSubAecswZE+Sm8jkvEa7kQgTPVRvwL/nd0E=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.1.1 h1:mkaGMgFkpDJVs7QUQrHvqEEpJFvoDrqGaHqMkywhGN0=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.1.1/go.mod h1:3S0vo7Y+O3Fjnnon5JXVrlG2IrfQkXasvKWB4OwX1lk=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.1.3 h1:RxiW5e1f3kgm6WGsnaRcnF1BOI+RnbwGKLSz+KPbxGY=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.1.3/go.mod h1:X6kh3l8pYjkOCz+PNLSrlyAhQSU/Cmkdnx72LwRwWZI=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
|
||||
github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda h1:NOo6+gM9NNPJ3W56nxOKb4164LEw094U0C8zYQM8mQU=
|
||||
|
|
@ -63,8 +63,8 @@ github.com/radovskyb/watcher v1.0.7/go.mod h1:78okwvY5wPdzcb1UYnip1pvrZNIVEIh/Cm
|
|||
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
|
||||
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ=
|
||||
github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=
|
||||
github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk=
|
||||
github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0=
|
||||
github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
|
||||
github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
|
||||
github.com/sevlyar/go-daemon v0.1.6 h1:EUh1MDjEM4BI109Jign0EaknA2izkOyi0LV3ro3QQGs=
|
||||
|
|
@ -77,8 +77,8 @@ github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0=
|
|||
github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
|
||||
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
|
||||
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ=
|
||||
github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk=
|
||||
github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI=
|
||||
github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
|
|
@ -101,8 +101,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
|
|||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
|
||||
golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
|
||||
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM=
|
||||
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc=
|
||||
golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY=
|
||||
golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI=
|
||||
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
#!/bin/bash
|
||||
work_dir=$(echo $1 | sed 's:/*$::')
|
||||
version="1.22.1"
|
||||
version="1.22.4"
|
||||
arch=`hostnamectl | grep "Arch" | rev | cut -d " " -f 1 | rev`
|
||||
|
||||
if [ $arch != "arm64" ]
|
||||
|
|
@ -13,3 +13,4 @@ wget "https://golang.org/dl/go$version.linux-$arch.tar.gz" -P "$work_dir"
|
|||
sudo rm -rf /usr/local/go
|
||||
sudo tar -C /usr/local -xzf "$work_dir"/go"$version".linux-$arch.tar.gz
|
||||
sudo ln -sf /usr/local/go/bin/go /usr/bin/go
|
||||
sudo ln -sf /usr/local/go/bin/gofmt /usr/bin/gofmt
|
||||
|
|
|
|||
|
|
@ -281,7 +281,7 @@ func (suite *dirTestSuite) TestDirGetStats() {
|
|||
// # Change mod of directory
|
||||
func (suite *dirTestSuite) TestDirChmod() {
|
||||
if suite.adlsTest == true {
|
||||
dirName := suite.testPath + "/test3"
|
||||
dirName := suite.testPath + "/testchmod"
|
||||
err := os.Mkdir(dirName, 0777)
|
||||
suite.Equal(nil, err)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,36 +0,0 @@
|
|||
logging:
|
||||
level: log_debug
|
||||
file-path: "blobfuse2-logs.txt"
|
||||
type: base
|
||||
|
||||
components:
|
||||
- libfuse
|
||||
- file_cache
|
||||
- attr_cache
|
||||
- azstorage
|
||||
|
||||
libfuse:
|
||||
attribute-expiration-sec: 0
|
||||
entry-expiration-sec: 0
|
||||
negative-entry-expiration-sec: 0
|
||||
ignore-open-flags: true
|
||||
|
||||
file_cache:
|
||||
path: { 1 }
|
||||
policy: lfu
|
||||
max-size-mb: 2048
|
||||
allow-non-empty-temp: true
|
||||
cleanup-on-start: true
|
||||
|
||||
attr_cache:
|
||||
timeout-sec: 3600
|
||||
|
||||
azstorage:
|
||||
type: { ACCOUNT_TYPE }
|
||||
endpoint: { ACCOUNT_ENDPOINT }
|
||||
use-http: false
|
||||
account-name: { NIGHTLY_STO_ACC_NAME }
|
||||
account-key: { NIGHTLY_STO_ACC_KEY }
|
||||
mode: key
|
||||
container: { 0 }
|
||||
tier: hot
|
||||
Загрузка…
Ссылка в новой задаче