Vibhansa/filenameencode (#525)

* Set retry policy to explonential for adls account

* Encode % in the file name

* Removing temp file

* Print permission string in case of failure

* Do not trucate file if the size remains same

* acl.Permissions returning extra bits for sticky. Ignore these bits as blobfuse does not support them

* Make % encoding in file name to be a configurable option

* Run multi special char file name test only for adls

* Do not upload if truncate results in same file size or truncate is setting size to 0 for a new file

* Maintain last upload time to stop double uploads in case of truncate and flush

* Encode % in url for folder name and other things

* no need to encode prefix seperatly as now entire url will be encoded

* Unused reference removed

* Correcting nightly yml error

* Increasing timeout for nightly

* Run multi special char file test for block blob as well

* Add test case of directory with special characters

* Search the file under directory where it was created

* Add file cache timeout to be 0 in feature_test

* Add file cache timeout to be 0 in feature_test

* Correcting failing rename test cases in pytest

* Try deb for nightly

* Fuse installation for mounting on debs

* Install fuse to mount and test

* Install golang to run gotests

* Install golang to run gotests

* Sudo permissions added as go tests are failing

* pre cleanup to unmount and deleted mount directory in case last run failed

* code correction

* Pytest failure to move file due to non closure

* nightly express test correction

* Add Rhel 7.5 to express test

* Adding recursive dir deletion case

* Pipeline correction

* Adding more flavours to nightly

* correcting recursive dir deletion case

* Checkout branch code added

* Adding suse to nightly

* golang install in suse

* Print curl version before mount

* Adding ubuntu 16-18-20 to express test

* No need for cleanup in case of ubuntu

* test1

* Making pipeline more configurable

* Rename parameter

* condition correction

* Adding rhel 7.5 with old curl

* Adding rhel 7.5 with old curl

* Instead of last update time watch for create or write call to decide whether to upload the file on flush or not

Co-authored-by: vibhansa-msft <>

blobfuse-artifacts.yml

@@ -163,8 +163,6 @@ stages:
 
         - script: |
             pwd
-            #sudo rm -rf $(Agent.BuildDirectory)
-            #sudo rm -rf $(Build.SourcesDirectory)
             cd /`pwd | cut -d '/' -f 2,3,4,5`
             sudo rm -rf [0-9]
           displayName: 'Clean Agent Directories'

blobfuse-nightly.yml

@@ -13,16 +13,16 @@ schedules:
   always: true
 
 parameters:
-- name: quick_test
-  displayName: 'Express Test'
+- name: exhaustive_test
+  displayName: 'Exhaustive Test'
   type: boolean
-  default: false
+  default: true
 
 # List of jobs to run in parallel
 jobs:
-  - ${{ if eq(parameters.quick_test, false) }}:
-    - job: NightlySanity_Ubuntu
-      timeoutInMinutes: 240
+  - ${{ if true }}:
+    - job: Set_1
+      timeoutInMinutes: 300
 
       # Define on what all OS we want to run this Pipeline
       strategy:
@@ -365,7 +365,7 @@ jobs:
       - script:  |
           sudo rm -rf $(MOUNT_DIR)/*
           sudo rm -rf $(TEMP_DIR)/*
-          ./build/blobfuse $(MOUNT_DIR) --tmp-path=$(TEMP_DIR) --config-file=$(BLOBFUSE_CFG_GTEST)
+          ./build/blobfuse $(MOUNT_DIR) --tmp-path=$(TEMP_DIR) --config-file=$(BLOBFUSE_CFG_GTEST) --file-cache-timeout-in-seconds=0
           cd test
           set -e && go test -v feature_test.go -args -mnt-path=$(MOUNT_DIR)
           cd -
@@ -461,7 +461,7 @@ jobs:
       - script:  |
           sudo rm -rf $(MOUNT_DIR)/*
           sudo rm -rf $(TEMP_DIR)/*
-          ./build/blobfuse $(MOUNT_DIR) --tmp-path=$(TEMP_DIR) --config-file=$(BLOBFUSE_CFG_ADLS) --use-adls=true
+          ./build/blobfuse $(MOUNT_DIR) --tmp-path=$(TEMP_DIR) --config-file=$(BLOBFUSE_CFG_ADLS) --use-adls=true --file-cache-timeout-in-seconds=0
           pidof blobfuse
           cd test
           set -e && go test -v feature_test.go -args -mnt-path=$(MOUNT_DIR) -adls=true
@@ -648,29 +648,53 @@ jobs:
     
 
 
-  - ${{ if eq(parameters.quick_test, true) }}:
+  - ${{ if eq(parameters.exhaustive_test, true) }}:
     # Other then Ubuntu just run the feature_test
-    - job: NightlySanity_RHEL
+    - job: Set_2
       timeoutInMinutes: 240
       strategy:
         matrix:
-          RHEL-7.5:
+          RHEL-7.5-new-curl:
             DistroVer: "RHEL-7.5"
             AgentName : "RHEL 7.5"
-            Description: "Red Hat Enterprise Linux 7.5"
+            Description: "Red Hat Enterprise Linux 7.5 New Curl"
             containerName: 'test-cnt-rhel-75'
 
-          #RHEL-8.1:
-          #  DistroVer: "RHEL-8.1"
-          #  AgentName: "RHEL 8.1"
-          #  Description: "Red Hat Enterprise Linux 8.1"
-          #  containerName: 'test-cnt-rhel-81'
+          RHEL-7.5-old-curl:
+            DistroVer: "RHEL-7.5"
+            AgentName : "RHEL 7.5.2"
+            Description: "Red Hat Enterprise Linux 7.5 Old Curl"
+            containerName: 'test-cnt-rhel-75-test'
+            
+          RHEL-8.1:
+            DistroVer: "RHEL-8.1"
+            AgentName: "RHEL 8.1"
+            Description: "Red Hat Enterprise Linux 8.1"
+            containerName: 'test-cnt-rhel-81'
 
-          #RHEL-8.2:
-          #  DistroVer: "RHEL-8.2"
-          #  AgentName: "RHEL 8.2"
-          #  Description: "Red Hat Enterprise Linux 8.2"
-          #  containerName: 'test-cnt-rhel-82'
+          RHEL-8.2:
+            DistroVer: "RHEL-8.2"
+            AgentName: "RHEL 8.2"
+            Description: "Red Hat Enterprise Linux 8.2"
+            containerName: 'test-cnt-rhel-82'
+
+          CentOS-7.0:
+            DistroVer: "CentOS-7.0"
+            AgentName: "COS 7.0"
+            Description: "CentOS Linux 7.0"
+            containerName: 'test-cnt-cent-7'
+
+          CentOS-8.0:
+            DistroVer: "CentOS-8.0"
+            AgentName: "COS 8.0"
+            Description: "CentOS Linux 8.0"
+            containerName: 'test-cnt-cent-8'
+
+          Oracle-8.1:
+            DistroVer: "Oracle-8.1"
+            AgentName: "ORA 8.1"
+            Description: "Oracle Linux 8.1 Gen 2"
+            containerName: 'test-cnt-ora-81'
 
       pool:
         name: "Blobfuse Pool"
@@ -733,6 +757,13 @@ jobs:
           displayName: "Checkout code"
           workingDirectory: $(WORK_DIR)
 
+        # Checkout branch
+        - script: |
+            echo $(Build.SourceBranch)
+            echo "Going for checkout of " `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`  
+            git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`   
+          displayName: "Checkout Branch"
+          workingDirectory: $(ROOT_DIR)
 
         # List commits consumed for this build
         - script: |
@@ -780,10 +811,11 @@ jobs:
 
         # Mount a directory for feature test (Block) and run it
         - script:  |
+            curl --version
             sudo fusermount -u $(MOUNT_DIR)
             sudo rm -rf $(MOUNT_DIR)/*
             sudo rm -rf $(TEMP_DIR)/*
-            ./build/blobfuse $(MOUNT_DIR) --tmp-path=$(TEMP_DIR) --config-file=$(BLOBFUSE_CFG_FILE)
+            ./build/blobfuse $(MOUNT_DIR) --tmp-path=$(TEMP_DIR) --config-file=$(BLOBFUSE_CFG_FILE) --file-cache-timeout-in-seconds=0 --use-adls=true --use-attr-cache=true
           workingDirectory: $(ROOT_DIR)
           displayName: "Mount Blobfuse"
 
@@ -792,7 +824,7 @@ jobs:
             cd test
             go test -v feature_test.go -args -mnt-path=$(MOUNT_DIR) -adls=true
           workingDirectory: $(ROOT_DIR)
-          displayName: TEST - BLOCK FeatureTest
+          displayName: TEST - ADLS FeatureTest
           timeoutInMinutes: 120
 
         - script:  |
@@ -814,3 +846,329 @@ jobs:
             sudo rm -rf [0-9]
           displayName: 'Clean Agent Directories'
           condition: always()
+
+    - job: Set_3
+      timeoutInMinutes: 240
+      strategy:
+        matrix:
+          Debian-9.0:
+            DistroVer: "Debian9.0"
+            AgentName : "DEB 9.0"
+            Description: "Debian Linux 9.0"
+            containerName: 'test-cnt-deb-9'
+          Debian-10.0:
+            DistroVer: "Debian10.0"
+            AgentName : "DEB 10.0"
+            Description: "Debian Linux 10.0 Gen 1"
+            containerName: 'test-cnt-deb-10'
+
+      pool:
+        name: "Blobfuse Pool"
+        demands:
+        - Agent.Name -equals $(AgentName)
+
+      variables:
+      - group: NightlyBlobFuse
+      - name: WORK_DIR
+        value: "/home/vsts/work/"
+      - name: ROOT_DIR
+        value: "/home/vsts/work/azure-storage-fuse"
+      - name: skipComponentGovernanceDetection
+        value: true
+
+      - name: MOUNT_DIR
+        value: "/home/vsts/work/blob_mnt"
+      - name: TEMP_DIR
+        value: "/mnt/blobfusetmp"
+      - name: BLOBFUSE_CFG_FILE
+        value: "/home/vsts/blobfuse.cfg"
+      
+      steps:
+        - checkout: none
+
+        # Install all dependencies
+        - script: |
+            echo $(Description)
+            hostnamectl
+          displayName: "Print Agent Info"
+
+        # Install all dependencies
+        - script: |
+            sudo apt-get update --fix-missing
+            sudo apt-get install pkg-config cmake libcurl4-gnutls-dev libgnutls28-dev uuid-dev libgcrypt20-dev libboost-all-dev gcc g++ golang -y
+          displayName: "Basic Tools Setup"
+
+        # Install libfuse
+        - script: |
+            sudo apt-get install libfuse-dev fuse -y
+          displayName: "libFuse Setup"
+
+        # Prebuild cleanup
+        - script: |
+            sudo fusermount -u $(MOUNT_DIR)
+            sudo kill -9 `pidof blobfuse`
+            sudo rm -rf $(WORK_DIR)
+          displayName: "Prebuild cleanup"
+
+        # Checkout code
+        - script: |
+            sudo mkdir -p $(WORK_DIR)
+            sudo chown -R `whoami` $(WORK_DIR)
+            chmod 777 $(WORK_DIR)
+          displayName: "Create Directory structure"
+
+        # Checkout code
+        - script: |
+            git clone https://github.com/Azure/azure-storage-fuse
+          displayName: "Checkout code"
+          workingDirectory: $(WORK_DIR)
+
+        # Checkout branch
+        - script: |
+            echo $(Build.SourceBranch)
+            echo "Going for checkout of " `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`  
+            git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`   
+          displayName: "Checkout Branch"
+          workingDirectory: $(ROOT_DIR)
+
+        # List commits consumed for this build
+        - script: |
+            echo "Below commits were done in last 12 hours : " `date` 
+            git --no-pager log --since="12 hours ago" --stat
+          displayName: "List Commits"
+          workingDirectory: $(ROOT_DIR)
+
+        # Build the blobFuse binary
+        - script: |
+            sudo chmod +x build.sh
+            . ./build.sh
+          displayName: "Build BlobFuse"
+          workingDirectory: $(ROOT_DIR)
+
+        # Test the binary just to make sure it was built fine
+        - script: |
+            ./build/blobfuse --version
+          displayName: "Test the binary"
+          workingDirectory: $(ROOT_DIR)
+
+        # Add the test work here
+        # Create directory structures for the basic testing
+        - script: |
+            sudo mkdir $(MOUNT_DIR)
+            sudo chown -R `whoami` $(MOUNT_DIR)
+            chmod 777 $(MOUNT_DIR)
+            sudo mkdir $(TEMP_DIR)
+            sudo chown -R `whoami` $(TEMP_DIR)
+            chmod 777 $(TEMP_DIR)
+          displayName: "Create Directories"
+        
+        # --- Create config files for testing
+        - script: |
+            sudo touch $(BLOBFUSE_CFG_FILE)
+            sudo chown -R `whoami` $(BLOBFUSE_CFG_FILE)
+            chmod 777 $(BLOBFUSE_CFG_FILE)
+            echo "accountName " $(NIGHTLY_STO_ACC_NAME) > $(BLOBFUSE_CFG_FILE)
+            echo "accountKey " $(NIGHTLY_STO_ACC_KEY) >> $(BLOBFUSE_CFG_FILE)
+            echo "authType Key" >> $(BLOBFUSE_CFG_FILE)
+            echo "containerName " $(containerName) >> $(BLOBFUSE_CFG_FILE)
+            cat $(BLOBFUSE_CFG_FILE)
+            echo "Account : " $(NIGHTLY_STO_ACC_NAME)
+          displayName: "Create config file"
+
+        # Mount a directory for feature test (Block) and run it
+        - script:  |
+            curl --version
+            sudo fusermount -u $(MOUNT_DIR)
+            sudo rm -rf $(MOUNT_DIR)/*
+            sudo rm -rf $(TEMP_DIR)/*
+            ./build/blobfuse $(MOUNT_DIR) --tmp-path=$(TEMP_DIR) --config-file=$(BLOBFUSE_CFG_FILE) --file-cache-timeout-in-seconds=0 --use-adls=true --use-attr-cache=true
+          workingDirectory: $(ROOT_DIR)
+          displayName: "Mount Blobfuse"
+
+        - script: | 
+            sudo rm -rf $(MOUNT_DIR)/*  
+            cd test
+            go test -v feature_test.go -args -mnt-path=$(MOUNT_DIR) -adls=true
+          workingDirectory: $(ROOT_DIR)
+          displayName: TEST - ADLS FeatureTest
+          timeoutInMinutes: 120
+
+        - script:  |
+            sudo rm -rf $(MOUNT_DIR)/*
+            sudo fusermount -u $(MOUNT_DIR)
+            sudo kill -9 `pidof blobfuse`
+            sudo rm -rf $(TEMP_DIR)/
+            sudo rm -rf $(MOUNT_DIR)
+            sudo rm -rf $(BLOBFUSE_CFG_FILE)
+            sudo rm -rf $(WORK_DIR)
+          workingDirectory: $(ROOT_DIR)
+          displayName: 'Cleanup Directories'
+          timeoutInMinutes: 30
+          condition: always()
+
+        - script: |
+            pwd
+            cd /`pwd | cut -d '/' -f 2,3,4,5`
+            sudo rm -rf [0-9]
+          displayName: 'Clean Agent Directories'
+          condition: always()
+
+    - job: Set_4
+      timeoutInMinutes: 240
+      strategy:
+        matrix:
+          SUSE-15G2:
+            DistroVer: "Suse-15Gen2"
+            AgentName : "SUSE 15G2"
+            Description: "SUSE Ent Linux 15-SP1-Gen2"
+            containerName: 'test-cnt-suse-15'
+
+      pool:
+        name: "Blobfuse Pool"
+        demands:
+        - Agent.Name -equals $(AgentName)
+
+      variables:
+      - group: NightlyBlobFuse
+      - name: WORK_DIR
+        value: "/home/vsts/work/"
+      - name: ROOT_DIR
+        value: "/home/vsts/work/azure-storage-fuse"
+      - name: skipComponentGovernanceDetection
+        value: true
+
+      - name: MOUNT_DIR
+        value: "/home/vsts/work/blob_mnt"
+      - name: TEMP_DIR
+        value: "/mnt/blobfusetmp"
+      - name: BLOBFUSE_CFG_FILE
+        value: "/home/vsts/blobfuse.cfg"
+      
+      steps:
+        - checkout: none
+
+        # Install all dependencies
+        - script: |
+            echo $(Description)
+            hostnamectl
+          displayName: "Print Agent Info"
+
+        # Install all dependencies
+        - script: |
+            sudo zypper -n update git tar make cmake gcc gcc-c++ curl-devel gnutls-devel libuuid-devel boost-devel libboost*devel libgcrypt-devel rpm-build go
+          displayName: "Basic Tools Setup"
+
+        # Install libfuse
+        - script: |
+            sudo zypper -n update fuse-devel fuse
+          displayName: "libFuse Setup"
+
+        # Prebuild cleanup
+        - script: |
+            sudo fusermount -u $(MOUNT_DIR)
+            sudo kill -9 `pidof blobfuse`
+            sudo rm -rf $(WORK_DIR)
+          displayName: "Prebuild cleanup"
+
+        # Checkout code
+        - script: |
+            sudo mkdir -p $(WORK_DIR)
+            sudo chown -R `whoami` $(WORK_DIR)
+            chmod 777 $(WORK_DIR)
+          displayName: "Create Directory structure"
+
+        # Checkout code
+        - script: |
+            git clone https://github.com/Azure/azure-storage-fuse
+          displayName: "Checkout code"
+          workingDirectory: $(WORK_DIR)
+
+        # Checkout branch
+        - script: |
+            echo $(Build.SourceBranch)
+            echo "Going for checkout of " `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`  
+            git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`   
+          displayName: "Checkout Branch"
+          workingDirectory: $(ROOT_DIR)
+
+        # List commits consumed for this build
+        - script: |
+            echo "Below commits were done in last 12 hours : " `date` 
+            git --no-pager log --since="12 hours ago" --stat
+          displayName: "List Commits"
+          workingDirectory: $(ROOT_DIR)
+
+        # Build the blobFuse binary
+        - script: |
+            sudo chmod +x build.sh
+            . ./build.sh
+          displayName: "Build BlobFuse"
+          workingDirectory: $(ROOT_DIR)
+
+        # Test the binary just to make sure it was built fine
+        - script: |
+            ./build/blobfuse --version
+          displayName: "Test the binary"
+          workingDirectory: $(ROOT_DIR)
+
+        # Add the test work here
+        # Create directory structures for the basic testing
+        - script: |
+            sudo mkdir $(MOUNT_DIR)
+            sudo chown -R `whoami` $(MOUNT_DIR)
+            chmod 777 $(MOUNT_DIR)
+            sudo mkdir $(TEMP_DIR)
+            sudo chown -R `whoami` $(TEMP_DIR)
+            chmod 777 $(TEMP_DIR)
+          displayName: "Create Directories"
+        
+        # --- Create config files for testing
+        - script: |
+            sudo touch $(BLOBFUSE_CFG_FILE)
+            sudo chown -R `whoami` $(BLOBFUSE_CFG_FILE)
+            chmod 777 $(BLOBFUSE_CFG_FILE)
+            echo "accountName " $(NIGHTLY_STO_ACC_NAME) > $(BLOBFUSE_CFG_FILE)
+            echo "accountKey " $(NIGHTLY_STO_ACC_KEY) >> $(BLOBFUSE_CFG_FILE)
+            echo "authType Key" >> $(BLOBFUSE_CFG_FILE)
+            echo "containerName " $(containerName) >> $(BLOBFUSE_CFG_FILE)
+            cat $(BLOBFUSE_CFG_FILE)
+            echo "Account : " $(NIGHTLY_STO_ACC_NAME)
+          displayName: "Create config file"
+
+        # Mount a directory for feature test (Block) and run it
+        - script:  |
+            curl --version
+            sudo fusermount -u $(MOUNT_DIR)
+            sudo rm -rf $(MOUNT_DIR)/*
+            sudo rm -rf $(TEMP_DIR)/*
+            ./build/blobfuse $(MOUNT_DIR) --tmp-path=$(TEMP_DIR) --config-file=$(BLOBFUSE_CFG_FILE) --file-cache-timeout-in-seconds=0 --use-adls=true --use-attr-cache=true
+          workingDirectory: $(ROOT_DIR)
+          displayName: "Mount Blobfuse"
+
+        - script: | 
+            sudo rm -rf $(MOUNT_DIR)/*  
+            cd test
+            go test -v feature_test.go -args -mnt-path=$(MOUNT_DIR) -adls=true
+          workingDirectory: $(ROOT_DIR)
+          displayName: TEST - ADLS FeatureTest
+          timeoutInMinutes: 120
+
+        - script:  |
+            sudo rm -rf $(MOUNT_DIR)/*
+            sudo fusermount -u $(MOUNT_DIR)
+            sudo kill -9 `pidof blobfuse`
+            sudo rm -rf $(TEMP_DIR)/
+            sudo rm -rf $(MOUNT_DIR)
+            sudo rm -rf $(BLOBFUSE_CFG_FILE)
+            sudo rm -rf $(WORK_DIR)
+          workingDirectory: $(ROOT_DIR)
+          displayName: 'Cleanup Directories'
+          timeoutInMinutes: 30
+          condition: always()
+
+        - script: |
+            pwd
+            cd /`pwd | cut -d '/' -f 2,3,4,5`
+            sudo rm -rf [0-9]
+          displayName: 'Clean Agent Directories'
+          condition: always()

blobfuse/blobfuse.cpp

@@ -12,6 +12,9 @@
 #include <include/DataLakeBfsClient.h>
 #include <include/AttrCacheBfsClient.h>
 
+
+bool gEncodeFullFileName = true;
+
 const std::string log_ident = "blobfuse";
 struct cmdlineOptions cmd_options;
 struct configParams config_options;
@@ -46,6 +49,7 @@ const struct fuse_opt option_spec[] =
     OPTION("--max-concurrency=%s", concurrency),
     OPTION("--cache-size-mb=%s", cache_size_mb),
     OPTION("--empty-dir-check=%s", empty_dir_check),
+    OPTION("--encode-full-file-name=%s", encode_full_file_name),
     OPTION("--version", version),
     OPTION("-v", version),
     OPTION("--help", help),
@@ -865,6 +869,18 @@ int read_and_set_arguments(int argc, char *argv[], struct fuse_args *args)
         config_options.cacheSize = stoi(cache_size) * (unsigned long long)(1024l * 1024l);
     }
 
+    if(cmd_options.encode_full_file_name != NULL)
+    {
+        std::string encode_full_file_name(cmd_options.encode_full_file_name);
+        if(encode_full_file_name == "false")
+        {
+            gEncodeFullFileName = false;
+        } else if(encode_full_file_name == "true")
+        {
+            gEncodeFullFileName = true;
+        }
+    }
+
     return 0;
 }
 

blobfuse/fileapis.cpp

@@ -37,6 +37,7 @@ int azs_open(const char *path, struct fuse_file_info *fi)
     struct stat buf;
     int statret = stat(mntPath, &buf);
     time_t now = time(NULL);
+    bool new_download = false;
     
 
     if (statret != 0 || 
@@ -114,6 +115,7 @@ int azs_open(const char *path, struct fuse_file_info *fi)
             new_time.modtime = last_modified;
             new_time.actime = 0;
             utime(mntPathString.c_str(), &new_time);
+            new_download = true;
         }
     }
 
@@ -139,18 +141,20 @@ int azs_open(const char *path, struct fuse_file_info *fi)
         return lock_result;
     }
     
-    if (!storage_client->isADLS()) {
-        fchmod(res, config_options.defaultPermission);
-    } else {
-        BfsFileProperty blob_property = storage_client->GetProperties(pathString.substr(1));
-        mode_t perms = blob_property.m_file_mode == 0 ?  config_options.defaultPermission : blob_property.m_file_mode;
-        fchmod(res, perms);
+    if (new_download) {
+        if (!storage_client->isADLS()) {
+            fchmod(res, config_options.defaultPermission);
+        } else {
+            BfsFileProperty blob_property = storage_client->GetProperties(pathString.substr(1));
+            mode_t perms = blob_property.m_file_mode == 0 ?  config_options.defaultPermission : blob_property.m_file_mode;
+            fchmod(res, perms);
 
-        // preserve the last modified time
-        struct utimbuf new_time;
-        new_time.modtime = blob_property.get_last_modified();
-        new_time.actime = blob_property.get_last_access(); 
-        utime(mntPathString.c_str(), &new_time); 
+            // preserve the last modified time
+            struct utimbuf new_time;
+            new_time.modtime = blob_property.get_last_modified();
+            new_time.actime = blob_property.get_last_access(); 
+            utime(mntPathString.c_str(), &new_time); 
+        }
     }
 
     // Store the open file handle, and whether or not the file should be uploaded on close().
@@ -221,6 +225,7 @@ int azs_create(const char *path, mode_t mode, struct fuse_file_info *fi)
 
     struct fhwrapper *fhwrap = new fhwrapper(res, true);
     fi->fh = (long unsigned int)fhwrap;
+    fhwrap->upload = true;
     syslog(LOG_INFO, "Successfully created file %s in file cache.\n", path);
     AZS_DEBUGLOGV("Returning success from azs_create with file %s.\n", path);
     return 0;
@@ -249,6 +254,7 @@ int azs_write(const char *path, const char *buf, size_t size, off_t offset, stru
     if (res == -1)
         res = -errno;
     g_gc_cache->addCacheBytes(path, size);
+    ((struct fhwrapper *)fi->fh)->upload = true;
     return res;
 }
 
@@ -295,7 +301,8 @@ int azs_flush(const char *path, struct fuse_file_info *fi)
         // For some file systems, however, close() flushes data, so we do want to do that before uploading data to a blob.
         // The solution (taken from the FUSE documentation) is to close a duplicate of the file descriptor.
         close(dup(((struct fhwrapper *)fi->fh)->fh));
-        if (((struct fhwrapper *)fi->fh)->upload)
+        if (((struct fhwrapper *)fi->fh)->write_mode && 
+            ((struct fhwrapper *)fi->fh)->upload)
         {
             // Here, we acquire the mutex on the file path.  This is necessary to guard against several race conditions.
             // For example, say that a cache refresh is triggered.  There is a small window of time where the file has been removed and not yet re-downloaded.
@@ -344,6 +351,7 @@ int azs_flush(const char *path, struct fuse_file_info *fi)
             else
             {
                 syslog(LOG_INFO, "Successfully uploaded file %s to blob %s.\n", path, blob_name.c_str());
+                ((struct fhwrapper *)fi->fh)->upload = false;
             }
             globalTimes.lastModifiedTime = time(NULL);
         } else {
@@ -374,6 +382,7 @@ int azs_release(const char *path, struct fuse_file_info * fi)
     // Close the file handle.
     // This must be done, even if the file no longer exists, otherwise we're leaking file handles.
     close(((struct fhwrapper *)fi->fh)->fh);
+    ((struct fhwrapper *)fi->fh)->upload = false;
 
 // TODO: Make this method resiliant to renames of the file (same way flush() is)
     std::string pathString(path);
@@ -491,11 +500,20 @@ int azs_truncate(const char * path, off_t off)
             return res;
         }
 
+        // If the file in cache has exactly the same size then there is no need to do trucate and upload
+        struct stat buf;
+        int statret = stat(mntPath, &buf);
+        if (statret == 0 && buf.st_size == off) {
+            azs_release(pathString.c_str(), &fi);
+            return 0;
+        }
+
         errno = 0;
         int truncret = truncate(mntPath, off);
         if (truncret == 0)
         {
             AZS_DEBUGLOGV("Successfully truncated file %s in the local file cache.", mntPath);
+            ((struct fhwrapper *)fi.fh)->upload = true;
             int flushret = azs_flush(pathString.c_str(), &fi);
             if(flushret != 0)
             {
@@ -523,6 +541,10 @@ int azs_truncate(const char * path, off_t off)
     int statret = stat(mntPath, &buf);
     if (statret == 0)
     {
+        if (buf.st_size == off) {
+            return 0;
+        }
+
         // The file exists in the local cache.  So, we call truncate() on the file in the cache, then upload a zero-length blob to the service, overriding any data.
         int truncret = truncate(mntPath, 0);
         if (truncret == 0)

blobfuse/include/BlobfuseGlobals.h

@@ -84,6 +84,7 @@ struct cmdlineOptions
     const char *concurrency; // Max Concurrency factor for blob client wrapper (default 40)
     const char *cache_size_mb; // MAX Size of cache in MBs
     const char *empty_dir_check;
+    const char *encode_full_file_name; // Encode the '%' symbol in file name
 };
 
 
@@ -93,7 +94,8 @@ struct fhwrapper
 {
     int fh; // The handle to the file in the file cache to use for read/write operations.
     bool upload; // True if the blob should be uploaded when the file is closed.  (False when the file was opened in read-only mode.)
-    fhwrapper(int fh, bool upload) : fh(fh), upload(upload)
+    bool write_mode;
+    fhwrapper(int fh, bool write) : fh(fh), upload(false), write_mode(write)
     {
 
     }

blobfuse/src/DataLakeBfsClient.cpp

@@ -40,6 +40,10 @@ bool DataLakeBfsClient::AuthenticateStorage()
         
     if(m_adls_client != NULL)
     {
+        if (m_adls_client->get_blob_client() && m_adls_client->get_blob_client()->context())
+            m_adls_client->get_blob_client()->context()->set_retry_policy(
+                        std::make_shared<azure::storage_lite::expo_retry_policy>());
+        
         //Authenticate the storage container by using a list call
         m_adls_client->list_paths_segmented(
                 configurations.containerName,
@@ -81,8 +85,6 @@ std::shared_ptr<adls_client_ext> DataLakeBfsClient::authenticate_adls_accountkey
                 configurations.concurrency,
                 false); //If this applies to blobs in the future, we can use this as a feature to exit
                                 // blobfuse if we run into anything unexpected instead of logging errors
-        syslog(LOG_DEBUG, "storage account urls: %s", account.get()->get_url(azure::storage_lite::storage_account::service::blob).to_string().c_str());
-
     }
     catch(const std::exception &ex)
     {

blobfuse/src/Permissions.cpp

@@ -36,8 +36,7 @@ mode_t aclToMode(access_control acl)
     }
     else if(permissions.size() != blobfuse_constants::acl_size)
     {
-        syslog(LOG_ERR, "Failure: Unexpected amount of permissions from service");
-        return mode;
+        syslog(LOG_DEBUG, "Unexpected amount of permissions from service : %s", permissions.c_str());
     }
     //try 
     {

cpplite/adls/include/adls_client.h

@@ -315,6 +315,11 @@ namespace azure { namespace storage_adls {
             return m_exception_enabled;
         }
 
+        AZURE_STORAGE_ADLS_API std::shared_ptr<azure::storage_lite::blob_client> get_blob_client()
+        {
+            return m_blob_client;
+        }
+
     protected:
         template<class RET, class FUNC>
         RET blob_client_adaptor(FUNC func);

cpplite/src/utility.cpp

@@ -18,6 +18,8 @@
 #include <vector>
 #include <algorithm>
 
+extern bool gEncodeFullFileName;
+
 namespace azure {  namespace storage_lite {
 
     std::string to_lowercase(std::string str)
@@ -217,6 +219,11 @@ namespace azure {  namespace storage_lite {
             }
             // Parameter path is already joint with '/'.
             ret['/'] = 1;
+
+            // Do not use % directly in path, if filename contains this then we need to encode
+            if (gEncodeFullFileName)
+                ret[37] = 0;
+
             return ret;
         }();
 
@@ -252,6 +259,11 @@ namespace azure {  namespace storage_lite {
 
             // Support '&' in file name like in 'A&b.txt'
             ret['&'] = 0;
+
+            // Do not use % directly in path, if filename contains this then we need to encode
+            if (gEncodeFullFileName)
+                ret[37] = 0;
+                
             return ret;
         }();
 

test/feature_test.go

@@ -59,7 +59,7 @@ func TestDirCreateSplChar(t *testing.T) {
 
 // # Create Directory with slash in name
 func TestDirCreateSlashChar(t *testing.T) {
-	dirName := mntPath + "PRQ\\STUV"
+	dirName := mntPath + "/" + "PRQ\\STUV"
 	err := os.Mkdir(dirName, 0777)
 	if err != nil {
 		t.Errorf("Failed to create directory : " + dirName + "(" + err.Error() + ")")
@@ -121,6 +121,11 @@ func TestDirMoveNonEmpty(t *testing.T) {
 		t.Errorf("Failed to create directory : " + dir3Name + "(" + err.Error() + ")")
 	}
 
+	err = os.Mkdir(dir3Name+"/abcdTest", 0777)
+	if err != nil {
+		t.Errorf("Failed to create directory : " + dir3Name + "/abcdTest (" + err.Error() + ")")
+	}
+
 	err = os.Rename(dir2Name, dir3Name+"/test2")
 	if err != nil {
 		t.Errorf("Failed to Move directory : " + dir2Name + "(" + err.Error() + ")")
@@ -138,7 +143,7 @@ func TestDirDeleteEmpty(t *testing.T) {
 
 // # Delete non-empty directory
 func TestDirDeleteNonEmpty(t *testing.T) {
-	dirName := mntPath + "/test3NE"
+	dirName := mntPath + "/test3NE/abcdTest"
 	err := os.Remove(dirName)
 	if err != nil {
 		if !strings.Contains(err.Error(), "directory not empty") {
@@ -149,8 +154,34 @@ func TestDirDeleteNonEmpty(t *testing.T) {
 
 // # Delete non-empty directory recursively
 func TestDirDeleteRecursive(t *testing.T) {
-	dirName := mntPath + "/test3NE"
-	err := os.RemoveAll(dirName)
+	dirName := mntPath + "/testREC"
+
+	err := os.Mkdir(dirName, 0777)
+	if err != nil {
+		t.Errorf("Failed to create directory : " + dirName + " (" + err.Error() + ")")
+	}
+
+	err = os.Mkdir(dirName+"/level1", 0777)
+	if err != nil {
+		t.Errorf("Failed to create directory : " + dirName + "/level1 (" + err.Error() + ")")
+	}
+
+	err = os.Mkdir(dirName+"/level2", 0777)
+	if err != nil {
+		t.Errorf("Failed to create directory : " + dirName + "/level2 (" + err.Error() + ")")
+	}
+
+	err = os.Mkdir(dirName+"/level1/l1", 0777)
+	if err != nil {
+		t.Errorf("Failed to create directory : " + dirName + "/level1/l1 (" + err.Error() + ")")
+	}
+
+	srcFile, err := os.OpenFile(dirName+"/level2/abc.txt", os.O_CREATE, 0777)
+	if err != nil {
+		t.Errorf("Failed to create file : " + dirName + "/level2/abc.txt (" + err.Error() + ")")
+	}
+	srcFile.Close()
+	err = os.RemoveAll(dirName)
 	if err != nil {
 		t.Errorf("Failed to delete directory recursively : " + dirName + "(" + err.Error() + ")")
 	}
@@ -238,7 +269,7 @@ func TestDirRenameFull(t *testing.T) {
 		t.Errorf("Failed to create directory : " + dirName + "(" + err.Error() + ")")
 	}
 
-	err = os.Mkdir(dirName + "/tmp", 0777)
+	err = os.Mkdir(dirName+"/tmp", 0777)
 	if err != nil {
 		t.Errorf("Failed to create directory : " + dirName + "/tmp (" + err.Error() + ")")
 	}
@@ -312,6 +343,87 @@ func TestFileCreateUtf8Char(t *testing.T) {
 	srcFile.Close()
 }
 
+func TestFileCreateMultiSpclCharWithinSpclDir(t *testing.T) {
+	speclChar := "abcd%23ABCD%34123-._~!$&'()*+,;=!@ΣΑΠΦΩ$भारत.txt"
+	speclDirName := mntPath + "/" + "abc%23%24%25efg-._~!$&'()*+,;=!@ΣΑΠΦΩ$भारत"
+	fileName := speclDirName + "/" + speclChar
+
+	err := os.Mkdir(speclDirName, 0777)
+	if err != nil {
+		t.Errorf("Failed to create directory " + speclDirName + " (" + err.Error() + ")")
+	}
+
+	//if adlsTest == true {
+	srcFile, err := os.OpenFile(fileName, os.O_CREATE, 0777)
+	if err != nil {
+		t.Errorf("Failed to create file " + fileName + " (" + err.Error() + ")")
+	}
+	srcFile.Close()
+	time.Sleep(time.Second * 2)
+
+	_, err = os.Stat(fileName)
+	if err != nil {
+		t.Errorf("Failed to get stat of file : " + fileName + "(" + err.Error() + ")")
+	}
+
+	files, err := ioutil.ReadDir(speclDirName)
+	if err != nil ||
+		len(files) < 1 {
+		t.Errorf("Failed to list directory : " + mntPath + "(" + err.Error() + ")")
+	}
+
+	found := false
+	for _, file := range files {
+		fmt.Println("######" + file.Name())
+		if file.Name() == speclChar {
+			fmt.Println("###### FOUND : " + file.Name())
+			found = true
+		}
+	}
+
+	if !found {
+		t.Errorf("Failed to find file with name " + speclChar)
+	}
+	//}
+}
+func TestFileCreateMultiSpclChar(t *testing.T) {
+	speclChar := "abcd%23ABCD%34123-._~!$&'()*+,;=!@ΣΑΠΦΩ$भारत.txt"
+	fileName := mntPath + "/" + speclChar
+
+	//if adlsTest == true {
+	srcFile, err := os.OpenFile(fileName, os.O_CREATE, 0777)
+	if err != nil {
+		t.Errorf("Failed to create file " + fileName + " (" + err.Error() + ")")
+	}
+	srcFile.Close()
+	time.Sleep(time.Second * 2)
+
+	_, err = os.Stat(fileName)
+	if err != nil {
+		t.Errorf("Failed to get stat of file : " + fileName + "(" + err.Error() + ")")
+	}
+
+	files, err := ioutil.ReadDir(mntPath)
+	if err != nil ||
+		len(files) < 1 {
+		t.Errorf("Failed to list directory : " + mntPath + "(" + err.Error() + ")")
+	}
+
+	found := false
+	for _, file := range files {
+		fmt.Println("######" + file.Name())
+		if file.Name() == speclChar {
+			fmt.Println("###### FOUND : " + file.Name())
+			found = true
+		}
+	}
+
+	if !found {
+		t.Errorf("Failed to find file with name " + speclChar)
+	}
+	//}
+}
+
 func TestFileCreateLongName(t *testing.T) {
 	fileName := mntPath + "Higher Call_ An Incredible True Story of Combat and Chivalry in the War-Torn Skies of World War II, A - Adam Makos & Larry Alexander.epub"
 
@@ -404,7 +516,7 @@ func TestFileNameConflict(t *testing.T) {
 	if err != nil {
 		t.Errorf("Failed to create file " + fileName + " (" + err.Error() + ")")
 	}
-	time.Sleep(3)
+	time.Sleep(time.Second * 3)
 }
 
 // # Copy file from once directory to another
@@ -528,7 +640,7 @@ func TestLinkCreate(t *testing.T) {
 	if err != nil {
 		t.Errorf("Failed to create symlink " + symName + " (" + err.Error() + ")")
 	}
-	time.Sleep(3)
+	time.Sleep(time.Second * 3)
 }
 
 // # Read a small file using symlink

test/tests.py

@@ -177,8 +177,10 @@ class RenameTests(BlobfuseTest):
         testFileNewName = "testFileMoved"
         testFileNewPath = os.path.join(self.blobstage, testFileNewName)
 
-        os.open(testFilePath, os.O_CREAT)
-        os.rename(testFilePath, testFileNewPath)
+        fd = os.open(testFilePath, os.O_CREAT)
+        os.close(fd)
+        os.system("mv " + testFilePath + " " + testFileNewPath)
+        #os.rename(testFilePath, testFileNewPath)
 
         with self.assertRaises(OSError) as e:
             os.stat(testFilePath)
@@ -198,7 +200,9 @@ class RenameTests(BlobfuseTest):
 
         os.mkdir(testDirPath)
         fd = os.open(testFilePath, os.O_CREAT)
-        os.rename(testFilePath, destFilePath)
+        os.close(fd)
+        os.system("mv " + testFilePath + " " + destFilePath)
+        #os.rename(testFilePath, destFilePath)
 
         self.validate_file_removal(testFilePath, testFileName, self.blobstage)
         self.validate_file_creation(destFilePath, testFileName, testDirPath)
@@ -216,7 +220,9 @@ class RenameTests(BlobfuseTest):
         fd = os.open(testFilePath, os.O_CREAT | os.O_WRONLY)
         testData = "test data"
         os.write(fd, testData.encode())
-        os.rename(testFilePath, destFilePath)
+        os.close(fd)
+        os.system("mv " + testFilePath + " " + destFilePath)
+        #os.rename(testFilePath, destFilePath)
 
         fd = os.open(destFilePath, os.O_RDONLY)
         data = os.read(fd, 20)