Depdendency upgrade for next release (#1486)

* Depdendency upgrade for next release
This commit is contained in:
Vikas Bhansali 2024-08-02 14:20:52 +05:30 коммит произвёл GitHub
Родитель bd0f411b46
Коммит 2c1781ec92
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: B5690EEEBB952194
6 изменённых файлов: 124 добавлений и 89 удалений

Просмотреть файл

@ -47,7 +47,7 @@ import (
// Standard config default values
const (
blobfuse2Version_ = "2.3.1~preview.1"
blobfuse2Version_ = "2.3.1"
DefaultMaxLogFileSize = 512
DefaultLogFileCount = 10

Просмотреть файл

@ -1070,6 +1070,7 @@ func (bc *BlockCache) getOrCreateBlock(handle *handlemap.Handle, offset uint64)
// If the block was staged earlier then we are overwriting it here so move it back to cooking queue
if block.flags.IsSet(BlockFlagSynced) {
log.Debug("BlockCache::getOrCreateBlock : Overwriting back to staged block %v for %v=>%s", block.id, handle.ID, handle.Path)
if block.node != nil {
_ = handle.Buffers.Cooked.Remove(block.node)
}
@ -1077,6 +1078,7 @@ func (bc *BlockCache) getOrCreateBlock(handle *handlemap.Handle, offset uint64)
block.node = handle.Buffers.Cooking.PushBack(block)
block.flags.Clear(BlockFlagSynced)
} else if block.flags.IsSet(BlockFlagDownloading) {
log.Debug("BlockCache::getOrCreateBlock : Waiting for download to finish for committed block %v for %v=>%s", block.id, handle.ID, handle.Path)
<-block.state
block.flags.Clear(BlockFlagDownloading)
block.Unblock()
@ -1144,11 +1146,11 @@ func (bc *BlockCache) lineupUpload(handle *handlemap.Handle, block *Block, listM
// if a block has data less than block size and is not the last block,
// add null at the end and upload the full block
if block.endIndex < uint64(handle.Size) {
log.Debug("BlockCache::lineupUpload : Appending null for block %v, size %v for %v=>%s", block.id, (block.endIndex - block.offset), handle.ID, handle.Path)
block.endIndex = block.offset + bc.blockSize
log.Debug("BlockCache::lineupUpload : Appending null for block %v", block.id)
} else if block.endIndex == uint64(handle.Size) {
// TODO: random write scenario where this block is not the last block
log.Debug("BlockCache::lineupUpload : Last block %v", block.id)
log.Debug("BlockCache::lineupUpload : Last block %v, size %v for %v=>%s", block.id, (block.endIndex - block.offset), handle.ID, handle.Path)
}
// id := listMap[block.id]

18
go.mod
Просмотреть файл

@ -3,10 +3,10 @@ module github.com/Azure/azure-storage-fuse/v2
go 1.20
require (
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2
github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.1.3
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0
github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.2.0
github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda
github.com/fsnotify/fsnotify v1.7.0
github.com/golang/mock v1.6.0
@ -27,7 +27,7 @@ require (
)
require (
github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
@ -49,10 +49,10 @@ require (
github.com/spf13/cast v1.6.0 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/crypto v0.24.0 // indirect
golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect
golang.org/x/net v0.26.0 // indirect
golang.org/x/sys v0.21.0 // indirect
golang.org/x/crypto v0.25.0 // indirect
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
golang.org/x/net v0.27.0 // indirect
golang.org/x/sys v0.22.0 // indirect
golang.org/x/text v0.16.0 // indirect
)

38
go.sum
Просмотреть файл

@ -1,14 +1,14 @@
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0 h1:1nGuui+4POelzDwI7RG56yfQJHCnKvwfMoU7VsEp+Zg=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0/go.mod h1:99EvauvlcJ1U06amZiksfYz/3aFGyIhWGHVyiZXtBAI=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0 h1:U2rTu3Ef+7w9FHKIAXM6ZyqF3UOWJZ12zIm8zECAFfg=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.0 h1:H+U3Gk9zY56G3u872L82bk4thcsy2Gghb9ExT4Zvm1o=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.0/go.mod h1:mgrmMSgaLp9hmax62XQTd0N4aAqSE5E0DulSpVYK7vc=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2 h1:YUUxeiOWgdAQE3pXt2H7QXzZs0q8UBjgRbl56qo8GYM=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2/go.mod h1:dmXQgZuiSubAecswZE+Sm8jkvEa7kQgTPVRvwL/nd0E=
github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.1.3 h1:RxiW5e1f3kgm6WGsnaRcnF1BOI+RnbwGKLSz+KPbxGY=
github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.1.3/go.mod h1:X6kh3l8pYjkOCz+PNLSrlyAhQSU/Cmkdnx72LwRwWZI=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0 h1:GJHeeA2N7xrG3q30L2UXDyuWRzDM900/65j70wcM4Ww=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0 h1:Be6KInmFEKV81c0pOAEbRYehLMwmmGI1exuFj248AMk=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0/go.mod h1:WCPBHsOXfBVnivScjs2ypRfimjEW0qPVLGgJkZlrIOA=
github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.2.0 h1:gXpwp0sGZz2FY9lVpSdM1rMpsP9PUtevHQyFhGoqHxY=
github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.2.0/go.mod h1:K+OqH/n5xyCEvbenN5OtZMycqHRCeoHh0whMoRjWYK4=
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU=
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda h1:NOo6+gM9NNPJ3W56nxOKb4164LEw094U0C8zYQM8mQU=
@ -99,16 +99,16 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY=
golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI=
golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@ -117,8 +117,8 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=

Просмотреть файл

@ -0,0 +1,20 @@
# Install esseentials
sudo apt-get install git fakeroot build-essential ncurses-dev xz-utils libssl-dev bc flex libelf-dev bison -y
# Get to mount path
cd $1
version=$2
# Download kernel tar ball
wget https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-$version.tar.xz
# Extract tarball
tar -xvf linux-$version.tar.xz
# Create default config
cd linux-$version
make defconfig
# build the kernel
make

Просмотреть файл

@ -1,69 +1,82 @@
# To create ramdisk
# sudo mkdir -p /mnt/ramdisk
# sudo chmod 777 /mnt/ramdisk
# sudo mount -t tmpfs -o rw,size=4G tmpfs /mnt/ramdisk
SERVICE="blobfuse2"
SCRIPT="longhaul.sh"
WORKDIR="/home/blobfuse/azure-storage-fuse"
# To create ramdisk
# sudo mount -t tmpfs -o rw,size=4G tmpfs /mnt/ramdisk
cd /home/vibhansa/go/src/azure-storage-fuse/
echo "Staring script"
if pgrep -x "$SERVICE" > /dev/null
then
if pgrep -x "$SCRIPT" > /dev/null
then
echo "`date` :: Already running" >> ./longhaul2.log
else
if [ `stat -c %s ./longhaul2.log` -gt 10485760 ]
then
echo "`date` :: Trimmed " > ./longhaul2.log
fi
echo "Check existing run"
#count=`ps -aux | grep $SCRIPT | wc -l`
#echo "Existing run count : $count"
echo "`whoami` : `date` :: `./blobfuse2 --version` Starting stress test " >> ./longhaul2.log
if [ -e "longhaul.lock" ]
then
echo "Script already running"
echo "`date` :: Already running" >> $WORKDIR/longhaul.log
else
touch longhaul.lock
echo "New script start"
if [ `stat -c %s $WORKDIR/longhaul.log` -gt 10485760 ]
then
echo "`date` :: Trimmed " > $WORKDIR/longhaul.log
fi
mem=$(top -b -n 1 -p `pgrep -x blobfuse2` | tail -1)
elap=$( ps -p `pgrep -x blobfuse2` -o etime | tail -1)
echo $mem " :: " $elap >> ./longhaul2.log
rm -rf /home/vibhansa/blob_mnt2/stress
rm -rf /home/vibhansa/blob_mnt2/myfile*
#go test -timeout 120m -v ./test/stress_test/stress_test.go -args -mnt-path=/home/vibhansa/blob_mnt2 -quick=false 2&> ./stress.log
./test/longhaul/stresstest.sh
echo "`whoami` : `date` :: Ending stress test " >> ./longhaul2.log
cp ./longhaul2.log /home/vibhansa/blob_mnt2/
cp ./stress.log /home/vibhansa/blob_mnt2/
sleep 30
echo "`whoami` : `date` :: `$WORKDIR/blobfuse2 --version` Starting test " >> $WORKDIR/longhaul.log
rm -rf /mnt/ramdisk/*
rm -rf /home/vibhansa/blob_mnt2/stress
sudo rm -rf /var/log/blob*.gz
fi
mem=$(top -b -n 1 -p `pgrep -x blobfuse2` | tail -1)
elap=$( ps -p `pgrep -x blobfuse2` -o etime | tail -1)
echo $mem " :: " $elap >> $WORKDIR/longhaul.log
echo "Delete old data"
echo "`date` : Cleanup old test data" >> $WORKDIR/longhaul.log
rm -rf /blob_mnt/kernel
echo "Start test"
echo "`date` : Building Kernel" >> $WORKDIR/longhaul.log
mkdir /blob_mnt/kernel
$WORKDIR/build_kernel.sh /blob_mnt/kernel/ 6.10.2
if [ $? -ne 0 ]; then
echo "`date` : Make Failed" >> $WORKDIR/longhaul.log
fi
echo "End test"
echo "`date` : Kernel Build complete" >> $WORKDIR/longhaul.log
sleep 30
echo "Cleanup post test"
rm -rf /blob_mnt/test/*
rm -rf /blob_mnt/kernel
cp $WORKDIR/longhaul.log /blob_mnt/
rm -rf longhaul.lock
fi
else
echo "`date` :: Re-Starting blobfuse2 *******************" >> ./longhaul2.log
rm -rf /home/vibhansa/blob_mnt2/*
rm -rf /mnt/ramdisk/*
sudo fusermount -u ~/blob_mnt2
rm -rf /mnt/ramdisk/*
./blobfuse2 mount ~/blob_mnt2 --config-file=./config.yaml
sleep 2
echo "Blobfuse not running"
echo "`date` :: Re-Starting blobfuse2 *******************" >> $WORKDIR/longhaul.log
$WORKDIR/blobfuse2 unmount all
if [ `stat -c %s ./restart2.log` -gt 10485760 ]
then
echo "`date` Trimmed " > ./restart2.log
fi
echo "`date`: Restart : `./blobfuse2 --version`" >> ./restart2.log
rm -rf /blob_mnt/*
# Send email that blobfuse2 has crashed
echo "Blobfuse2 Failure" | mail -s "Blobfuse2 Restart" -A ./restart2.log -a "From: longhaul@blobfuse.com" vibhansa@microsoft.com
cp /var/log/blobfuse2.log /home/vibhansa/blob_mnt2/
cp ./longhaul2.log /home/vibhansa/blob_mnt2/
cp ./restart2.log /home/vibhansa/blob_mnt2/
fi
export AZURE_STORAGE_ACCOUNT=vikasfuseblob
export AZURE_STORAGE_AUTH_TYPE=msi
export AZURE_STORAGE_IDENTITY_CLIENT_ID=1f1551d2-2db2-4d4d-a6f5-d7edbe75d98e
echo "Start blobfuse"
$WORKDIR/blobfuse2 mount /blob_mnt --log-level=log_debug --log-file-path=$WORKDIR/blobfuse2.log --log-type=base --block-cache --container-name=longhaul
sleep 2
if [ `stat -c %s $WORKDIR/restart.log` -gt 10485760 ]
then
echo "`date` Trimmed " > $WORKDIR/restart.log
fi
echo "`date`: Restart : `$WORKDIR/blobfuse2 --version`" >> $WORKDIR/restart.log
echo "Send mail"
# Send email that blobfuse2 has crashed
echo "Blobfuse2 Failure" | mail -s "Blobfuse2 Restart" -A $WORKDIR/restart.log -a "From: longhaul@blobfuse.com" vibhansa@microsoft.com
cp $WORKDIR/blobfuse2.log /blob_mnt/
cp $WORKDIR/longhaul.log /blob_mnt/
cp $WORKDIR/restart.log /blob_mnt/
fi