Depdendency upgrade for next release (#1486)

* Depdendency upgrade for next release

common/types.go

@@ -47,7 +47,7 @@ import (
 
 // Standard config default values
 const (
-	blobfuse2Version_ = "2.3.1~preview.1"
+	blobfuse2Version_ = "2.3.1"
 
 	DefaultMaxLogFileSize = 512
 	DefaultLogFileCount   = 10

component/block_cache/block_cache.go

@@ -1070,6 +1070,7 @@ func (bc *BlockCache) getOrCreateBlock(handle *handlemap.Handle, offset uint64)
 
 		// If the block was staged earlier then we are overwriting it here so move it back to cooking queue
 		if block.flags.IsSet(BlockFlagSynced) {
+			log.Debug("BlockCache::getOrCreateBlock : Overwriting back to staged block %v for %v=>%s", block.id, handle.ID, handle.Path)
 			if block.node != nil {
 				_ = handle.Buffers.Cooked.Remove(block.node)
 			}
@@ -1077,6 +1078,7 @@ func (bc *BlockCache) getOrCreateBlock(handle *handlemap.Handle, offset uint64)
 			block.node = handle.Buffers.Cooking.PushBack(block)
 			block.flags.Clear(BlockFlagSynced)
 		} else if block.flags.IsSet(BlockFlagDownloading) {
+			log.Debug("BlockCache::getOrCreateBlock : Waiting for download to finish for committed block %v for %v=>%s", block.id, handle.ID, handle.Path)
 			<-block.state
 			block.flags.Clear(BlockFlagDownloading)
 			block.Unblock()
@@ -1144,11 +1146,11 @@ func (bc *BlockCache) lineupUpload(handle *handlemap.Handle, block *Block, listM
 	// if a block has data less than block size and is not the last block,
 	// add null at the end and upload the full block
 	if block.endIndex < uint64(handle.Size) {
+		log.Debug("BlockCache::lineupUpload : Appending null for block %v, size %v for %v=>%s", block.id, (block.endIndex - block.offset), handle.ID, handle.Path)
 		block.endIndex = block.offset + bc.blockSize
-		log.Debug("BlockCache::lineupUpload : Appending null for block %v", block.id)
 	} else if block.endIndex == uint64(handle.Size) {
 		// TODO: random write scenario where this block is not the last block
-		log.Debug("BlockCache::lineupUpload : Last block %v", block.id)
+		log.Debug("BlockCache::lineupUpload : Last block %v, size %v for %v=>%s", block.id, (block.endIndex - block.offset), handle.ID, handle.Path)
 	}
 
 	// id := listMap[block.id]

go.mod

@@ -3,10 +3,10 @@ module github.com/Azure/azure-storage-fuse/v2
 go 1.20
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.1.3
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.2.0
 	github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda
 	github.com/fsnotify/fsnotify v1.7.0
 	github.com/golang/mock v1.6.0
@@ -27,7 +27,7 @@ require (
 )
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
@@ -49,10 +49,10 @@ require (
 	github.com/spf13/cast v1.6.0 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/crypto v0.24.0 // indirect
+	golang.org/x/crypto v0.25.0 // indirect
-	golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect
+	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
-	golang.org/x/net v0.26.0 // indirect
+	golang.org/x/net v0.27.0 // indirect
-	golang.org/x/sys v0.21.0 // indirect
+	golang.org/x/sys v0.22.0 // indirect
 	golang.org/x/text v0.16.0 // indirect
 )
 

go.sum

@@ -1,14 +1,14 @@
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0 h1:1nGuui+4POelzDwI7RG56yfQJHCnKvwfMoU7VsEp+Zg=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0 h1:GJHeeA2N7xrG3q30L2UXDyuWRzDM900/65j70wcM4Ww=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0/go.mod h1:99EvauvlcJ1U06amZiksfYz/3aFGyIhWGHVyiZXtBAI=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0 h1:U2rTu3Ef+7w9FHKIAXM6ZyqF3UOWJZ12zIm8zECAFfg=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.0 h1:H+U3Gk9zY56G3u872L82bk4thcsy2Gghb9ExT4Zvm1o=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.0/go.mod h1:mgrmMSgaLp9hmax62XQTd0N4aAqSE5E0DulSpVYK7vc=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2 h1:YUUxeiOWgdAQE3pXt2H7QXzZs0q8UBjgRbl56qo8GYM=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0 h1:Be6KInmFEKV81c0pOAEbRYehLMwmmGI1exuFj248AMk=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2/go.mod h1:dmXQgZuiSubAecswZE+Sm8jkvEa7kQgTPVRvwL/nd0E=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0/go.mod h1:WCPBHsOXfBVnivScjs2ypRfimjEW0qPVLGgJkZlrIOA=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.1.3 h1:RxiW5e1f3kgm6WGsnaRcnF1BOI+RnbwGKLSz+KPbxGY=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.2.0 h1:gXpwp0sGZz2FY9lVpSdM1rMpsP9PUtevHQyFhGoqHxY=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.1.3/go.mod h1:X6kh3l8pYjkOCz+PNLSrlyAhQSU/Cmkdnx72LwRwWZI=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.2.0/go.mod h1:K+OqH/n5xyCEvbenN5OtZMycqHRCeoHh0whMoRjWYK4=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
 github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda h1:NOo6+gM9NNPJ3W56nxOKb4164LEw094U0C8zYQM8mQU=
@@ -99,16 +99,16 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
+golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
-golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
+golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
-golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY=
+golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
-golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI=
+golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
+golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
-golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
+golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -117,8 +117,8 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
+golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
-golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=

test/longhaul/build_kernel.sh

@@ -0,0 +1,20 @@
+
+# Install esseentials
+sudo apt-get install git fakeroot build-essential ncurses-dev xz-utils libssl-dev bc flex libelf-dev bison -y
+
+# Get to mount path
+cd $1
+version=$2
+
+# Download kernel tar ball
+wget https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-$version.tar.xz
+
+# Extract tarball
+tar -xvf linux-$version.tar.xz
+
+# Create default config
+cd linux-$version
+make defconfig
+
+# build the kernel
+make

test/longhaul/longhaul.sh

@@ -1,69 +1,82 @@
-
-# To create ramdisk
-# sudo mkdir -p /mnt/ramdisk
-# sudo chmod 777 /mnt/ramdisk
-# sudo mount -t tmpfs -o rw,size=4G tmpfs /mnt/ramdisk
-
-
 SERVICE="blobfuse2"
 SCRIPT="longhaul.sh"
+WORKDIR="/home/blobfuse/azure-storage-fuse"
 
-# To create ramdisk
+echo "Staring script"
-# sudo mount -t tmpfs -o rw,size=4G tmpfs /mnt/ramdisk
-
-cd /home/vibhansa/go/src/azure-storage-fuse/
-
 if pgrep -x "$SERVICE" > /dev/null
 then
-	if pgrep -x "$SCRIPT" > /dev/null
+        echo "Check existing run"
-	then
+        #count=`ps -aux | grep $SCRIPT | wc -l`
-		echo "`date` :: Already running" >> ./longhaul2.log
+        #echo "Existing run count  : $count"
-	else
-		if [ `stat -c %s ./longhaul2.log` -gt 10485760 ]
-		then 
-			echo "`date` :: Trimmed " > ./longhaul2.log
-		fi
 
-		echo "`whoami` : `date` :: `./blobfuse2 --version` Starting stress test " >> ./longhaul2.log
+        if [ -e "longhaul.lock" ]
+        then
+                echo "Script already running"
+                echo "`date` :: Already running" >> $WORKDIR/longhaul.log
+        else
+                touch longhaul.lock
+                echo "New script start"
+                if [ `stat -c %s $WORKDIR/longhaul.log` -gt 10485760 ]
+                then
+                        echo "`date` :: Trimmed " > $WORKDIR/longhaul.log
+                fi
 
-		mem=$(top -b -n 1 -p `pgrep -x blobfuse2` | tail -1)
+                echo "`whoami` : `date` :: `$WORKDIR/blobfuse2 --version` Starting test " >> $WORKDIR/longhaul.log
-		elap=$( ps -p `pgrep -x blobfuse2` -o etime | tail -1)
-		echo $mem " :: " $elap >> ./longhaul2.log
-	
-		rm -rf /home/vibhansa/blob_mnt2/stress	
-		rm -rf /home/vibhansa/blob_mnt2/myfile*
-		
-		#go test -timeout 120m -v ./test/stress_test/stress_test.go -args -mnt-path=/home/vibhansa/blob_mnt2 -quick=false 2&> ./stress.log
-		./test/longhaul/stresstest.sh
-		echo "`whoami` : `date` :: Ending stress test " >> ./longhaul2.log
-		cp  ./longhaul2.log  /home/vibhansa/blob_mnt2/
-		cp ./stress.log /home/vibhansa/blob_mnt2/
-		
-		sleep 30
 
-		rm -rf /mnt/ramdisk/*
+                mem=$(top -b -n 1 -p `pgrep -x blobfuse2` | tail -1)
-		rm -rf /home/vibhansa/blob_mnt2/stress	
+                elap=$( ps -p `pgrep -x blobfuse2` -o etime | tail -1)
-		sudo rm -rf /var/log/blob*.gz
+                echo $mem " :: " $elap >> $WORKDIR/longhaul.log
-	fi
+
+                echo "Delete old data"
+                echo "`date` : Cleanup old test data" >> $WORKDIR/longhaul.log
+                rm -rf /blob_mnt/kernel
+
+                echo "Start test"
+                echo "`date` : Building Kernel"  >> $WORKDIR/longhaul.log
+                mkdir /blob_mnt/kernel
+                $WORKDIR/build_kernel.sh /blob_mnt/kernel/ 6.10.2
+
+                if [ $? -ne 0 ]; then
+                  echo "`date` : Make Failed" >> $WORKDIR/longhaul.log
+                fi
+                echo "End test"
+                echo "`date` : Kernel Build complete"  >> $WORKDIR/longhaul.log
+
+                sleep 30
+                echo "Cleanup post test"
+                rm -rf /blob_mnt/test/*
+                rm -rf /blob_mnt/kernel
+
+                cp  $WORKDIR/longhaul.log  /blob_mnt/
+                rm -rf longhaul.lock
+        fi
 else
-	echo "`date` :: Re-Starting blobfuse2 *******************" >> ./longhaul2.log
+        echo "Blobfuse not running"
-	rm -rf /home/vibhansa/blob_mnt2/*
+        echo "`date` :: Re-Starting blobfuse2 *******************" >> $WORKDIR/longhaul.log
-	rm -rf /mnt/ramdisk/*
+        $WORKDIR/blobfuse2 unmount all
-	sudo fusermount -u ~/blob_mnt2
-	rm -rf /mnt/ramdisk/*
-	./blobfuse2 mount ~/blob_mnt2 --config-file=./config.yaml
-	sleep 2
 
-	if [ `stat -c %s ./restart2.log` -gt 10485760 ]
+        rm -rf /blob_mnt/*
-	then 
-		echo "`date` Trimmed " > ./restart2.log
-	fi
-	echo "`date`: Restart : `./blobfuse2 --version`" >> ./restart2.log
 
-	# Send email that blobfuse2 has crashed
+        export AZURE_STORAGE_ACCOUNT=vikasfuseblob
-	echo "Blobfuse2 Failure" | mail -s "Blobfuse2 Restart" -A ./restart2.log -a "From: longhaul@blobfuse.com" vibhansa@microsoft.com
+        export AZURE_STORAGE_AUTH_TYPE=msi
-	
+        export AZURE_STORAGE_IDENTITY_CLIENT_ID=1f1551d2-2db2-4d4d-a6f5-d7edbe75d98e
-	cp /var/log/blobfuse2.log /home/vibhansa/blob_mnt2/
+
-	cp ./longhaul2.log  /home/vibhansa/blob_mnt2/
+        echo "Start blobfuse"
-	cp ./restart2.log /home/vibhansa/blob_mnt2/
+        $WORKDIR/blobfuse2 mount /blob_mnt --log-level=log_debug --log-file-path=$WORKDIR/blobfuse2.log --log-type=base --block-cache --container-name=longhaul
-fi	
+
+        sleep 2
+
+        if [ `stat -c %s $WORKDIR/restart.log` -gt 10485760 ]
+        then
+                echo "`date` Trimmed " > $WORKDIR/restart.log
+        fi
+        echo "`date`: Restart : `$WORKDIR/blobfuse2 --version`" >> $WORKDIR/restart.log
+
+        echo "Send mail"
+        # Send email that blobfuse2 has crashed
+        echo "Blobfuse2 Failure" | mail -s "Blobfuse2 Restart" -A $WORKDIR/restart.log -a "From: longhaul@blobfuse.com" vibhansa@microsoft.com
+
+        cp $WORKDIR/blobfuse2.log /blob_mnt/
+        cp $WORKDIR/longhaul.log  /blob_mnt/
+        cp $WORKDIR/restart.log /blob_mnt/
+fi