Merge with master and update README

README.md

@@ -60,10 +60,10 @@ For more information, see the [wiki](https://github.com/Azure/azure-storage-fuse
     * [OPTIONAL] **--cache-size-mb=1000** : option to setup the cache-size in MB. Default will be 80% of the available memory, eviction will happen beyond that. Use this option to lower the cache size or increase it. This option is only available after version 1.3.1.
      * [OPTIONAL] **--attr_timeout=20** : The attribute timeout in seconds. Performance improvement option. It is a default fuse option. For further details look at the FUSE man page. The attributes of recently accessed files will be saved for the specified seconds.
      * [OPTIONAL] **--entry_timeout=20** : The entry timeout in seconds. Performance improvement option. It is a default fuse option. For further details look at the FUSE man page. The attributes of recently accessed files will be saved for the specified seconds.
+     * [OPTIONAL] **--cancel-list-on-mount-seconds=0** : libFuse implicitly issues a list command on mount resulting into a call to container to retreive list of blobs. User can avoid this call by disabling the list feature for given number of seconds specified in this parameter. Default value is 0 meaning list call is not disabled.    
      * [OPTIONAL] **--high-disk-threshold=90** : High disk threshold percentage. When disk usage of cache directory reaches this mark start evicting the cache. This parameter overrides 'file-cache-timeout-in-seconds' parameter and cached file will be removed even if timeout is yet to expire. Files which are currently in use (open) will not be evicted from cache.
      * [OPTIONAL] **--low-disk-threshold=80** : Low disk threshold percentage. When disk usage of cache directory reaches high-disk-threshold mark start evicting the cache. Stop cache eviction when disk usage returns back to level specified by low-disk-threshold.
-     
-    
+     * [OPTIONAL] **--upload-modified-only=false** : If any file is open in 'write' mode then blobfuse upload that file on close, ir-respective of file was modified or not. Setting this flag to true means files will be uploaded only if they were modified after opening.
 ### Valid authentication setups:
 
 - Account Name & Key (`authType Key`)

blobfuse-nightly.yml

@@ -70,6 +70,9 @@ jobs:
         value: "./connection.cfg"
       - name: BLOBFUSE_STRESS_DIR
         value: "/home/vsts/work/blob_stress"
+      - name: DECODE_PERCENTS
+        value: false
+        
     
 
       # Each job has set of steps to be done

blobfuse/blobfuse.cpp

@@ -49,7 +49,8 @@ const struct fuse_opt option_spec[] =
     OPTION("--max-concurrency=%s", concurrency),
     OPTION("--cache-size-mb=%s", cache_size_mb),
     OPTION("--empty-dir-check=%s", empty_dir_check),
-    OPTION("--upload-if-modified=%s", upload_if_modified),
+    OPTION("--upload-modified-only=%s", upload_if_modified),
+    OPTION("--cancel-list-on-mount-seconds=%s", cancel_list_on_mount_seconds),
     OPTION("--high-disk-threshold=%s", high_disk_threshold),
     OPTION("--low-disk-threshold=%s", low_disk_threshold),
     OPTION("--version", version),
@@ -932,6 +933,12 @@ int read_and_set_arguments(int argc, char *argv[], struct fuse_args *args)
         config_options.cacheSize = stoi(cache_size) * (unsigned long long)(1024l * 1024l);
     }
 
+    config_options.cancel_list_on_mount_secs = 0;
+    if (cmd_options.cancel_list_on_mount_seconds != NULL) 
+    {
+        std::string cancel_list_on_mount_secs(cmd_options.cancel_list_on_mount_seconds);
+        config_options.cancel_list_on_mount_secs = stoi(cancel_list_on_mount_secs);
+    }
     // Make high and low disk threshold percentage a configurable option
     if (cmd_options.high_disk_threshold != NULL) 
     {
@@ -959,7 +966,6 @@ int read_and_set_arguments(int argc, char *argv[], struct fuse_args *args)
         config_options.low_disk_threshold = LOW_THRESHOLD_VALUE;
     }
     syslog(LOG_INFO, "Disk Thresholds : %d - %d", config_options.high_disk_threshold, config_options.low_disk_threshold);
-
     return 0;
 }
 

blobfuse/directoryapis.cpp

@@ -43,6 +43,12 @@ int azs_mkdir(const char *path, mode_t)
 int azs_readdir(const char *path, void *buf, fuse_fill_dir_t filler, off_t, struct fuse_file_info *)
 {
     AZS_DEBUGLOGV("azs_readdir called with path = %s\n", path);
+
+    if (config_options.cancel_list_on_mount_secs > 0) {
+        AZS_DEBUGLOGV("azs_readdir blocked by user config for %d more secs\n", config_options.cancel_list_on_mount_secs);
+        return 0;
+    }
+
     std::string pathStr(path);
     if (pathStr.size() > 1)
     {

blobfuse/include/BlobfuseGlobals.h

@@ -63,6 +63,7 @@ struct configParams
     int defaultPermission;
     int concurrency;
     unsigned long long cacheSize;
+    volatile int  cancel_list_on_mount_secs;
     bool emptyDirCheck;
     bool uploadIfModified;
     std::string mntPath;
@@ -87,6 +88,7 @@ struct cmdlineOptions
     const char *help; // print blobfuse usage
     const char *concurrency; // Max Concurrency factor for blob client wrapper (default 40)
     const char *cache_size_mb; // MAX Size of cache in MBs
+    const char *cancel_list_on_mount_seconds; // Block the list api call on mount for n seconds
     const char *empty_dir_check;
     const char *upload_if_modified;
     const char *encode_full_file_name; // Encode the '%' symbol in file name

blobfuse/src/BlobfuseConstants.cpp

@@ -9,7 +9,7 @@ namespace blobfuse_constants {
     const int max_concurrency_oauth = 2;
     const int max_retry_oauth = 5;
     const int max_concurrency_blob_wrapper = 40;
-    const int def_concurrency_blob_wrapper = 20;
+    const int def_concurrency_blob_wrapper = 12;
 
     const char* oauth_request_uri = "https://login.microsoftonline.com";
     const char* spn_request_path = "oauth2/v2.0/token";

blobfuse/src/BlobfuseGcCache.cpp

@@ -106,7 +106,11 @@ void gc_cache::run_gc_cache()
         if(is_empty)
         {
             //run it every 1 second
-            usleep(1000);
+            sleep(1);
+            if (config_options.cancel_list_on_mount_secs > 0) {
+                //AZS_DEBUGLOGV("Reducing lst block wait time %d\n", config_options.cancel_list_on_mount_secs);
+                config_options.cancel_list_on_mount_secs--;
+            }
             continue;
         }
 
@@ -184,7 +188,7 @@ void gc_cache::run_gc_cache()
         else
         {
             // no file was timed out - let's wait a second
-            usleep(1000);
+            sleep(1);
             //check disk space
             disk_threshold_reached = check_disk_space();
         }

blobfuse/src/BlockBlobBfsClient.cpp

@@ -148,7 +148,7 @@ std::shared_ptr<blob_client_wrapper> BlockBlobBfsClient::authenticate_blob_msi()
             true, //use_https must be true to use oauth
             configurations.blobEndpoint);
         std::shared_ptr<blob_client> blobClient =
-            std::make_shared<blob_client>(account, max_concurrency_oauth);
+            std::make_shared<blob_client>(account, configurations.concurrency);
         errno = 0;
         return std::make_shared<blob_client_wrapper>(blobClient);
     }
@@ -192,7 +192,7 @@ std::shared_ptr<blob_client_wrapper> BlockBlobBfsClient::authenticate_blob_spn()
             true, //use_https must be true to use oauth
             configurations.blobEndpoint);
         std::shared_ptr<blob_client> blobClient =
-            std::make_shared<blob_client>(account, max_concurrency_oauth);
+            std::make_shared<blob_client>(account, configurations.concurrency);
         errno = 0;
         return std::make_shared<blob_client_wrapper>(blobClient);
     }
@@ -212,7 +212,7 @@ std::shared_ptr<blob_client_wrapper> BlockBlobBfsClient::authenticate_blob_spn()
 void BlockBlobBfsClient::UploadFromFile(const std::string sourcePath, METADATA &metadata)
 {
     std::string blobName = sourcePath.substr(configurations.tmpPath.size() + 6 /* there are six characters in "/root/" */);
-    m_blob_client->upload_file_to_blob(sourcePath, configurations.containerName, blobName, metadata);
+    m_blob_client->upload_file_to_blob(sourcePath, configurations.containerName, blobName, metadata, configurations.concurrency);
     // upload_file_to_blob does not return a status or success if the blob succeeded
     // it does syslog if there was an exception and changes the errno.
 }
@@ -237,7 +237,7 @@ void BlockBlobBfsClient::UploadFromStream(std::istream &sourceStream, const std:
 ///<returns>none</returns>
 long int BlockBlobBfsClient::DownloadToFile(const std::string blobName, const std::string filePath, time_t &last_modified)
 {
-    m_blob_client->download_blob_to_file(configurations.containerName, blobName, filePath, last_modified);
+    m_blob_client->download_blob_to_file(configurations.containerName, blobName, filePath, last_modified, configurations.concurrency);
     struct stat stbuf;
     lstat(filePath.c_str(), &stbuf);
     if (0 == stat(filePath.c_str(), &stbuf))

blobfuse/src/OAuthTokenCredentialManager.cpp

@@ -125,7 +125,7 @@ void OAuthTokenCredentialManager::TokenMonitor()
                 // If we fail, explain ourselves and unlock.
                 syslog(LOG_ERR, "OAUTH Token : TokenMonitor : Failed to refresh (%s)\n", ex.what());
                 syslog(LOG_ERR, "OAUTH Token : TokenMonitor : Retry refreshing the token attempt %d", retry_count++);
-                usleep(10 * 1000);
+                usleep(100 * 1000);
             }
         }
         token_mutex.unlock();