diff --git a/README.md b/README.md index 3f3ed2cc..b0abed03 100755 --- a/README.md +++ b/README.md @@ -60,10 +60,10 @@ For more information, see the [wiki](https://github.com/Azure/azure-storage-fuse * [OPTIONAL] **--cache-size-mb=1000** : option to setup the cache-size in MB. Default will be 80% of the available memory, eviction will happen beyond that. Use this option to lower the cache size or increase it. This option is only available after version 1.3.1. * [OPTIONAL] **--attr_timeout=20** : The attribute timeout in seconds. Performance improvement option. It is a default fuse option. For further details look at the FUSE man page. The attributes of recently accessed files will be saved for the specified seconds. * [OPTIONAL] **--entry_timeout=20** : The entry timeout in seconds. Performance improvement option. It is a default fuse option. For further details look at the FUSE man page. The attributes of recently accessed files will be saved for the specified seconds. + * [OPTIONAL] **--cancel-list-on-mount-seconds=0** : libFuse implicitly issues a list command on mount resulting into a call to container to retreive list of blobs. User can avoid this call by disabling the list feature for given number of seconds specified in this parameter. Default value is 0 meaning list call is not disabled. * [OPTIONAL] **--high-disk-threshold=90** : High disk threshold percentage. When disk usage of cache directory reaches this mark start evicting the cache. This parameter overrides 'file-cache-timeout-in-seconds' parameter and cached file will be removed even if timeout is yet to expire. Files which are currently in use (open) will not be evicted from cache. * [OPTIONAL] **--low-disk-threshold=80** : Low disk threshold percentage. When disk usage of cache directory reaches high-disk-threshold mark start evicting the cache. Stop cache eviction when disk usage returns back to level specified by low-disk-threshold. - - + * [OPTIONAL] **--upload-modified-only=false** : If any file is open in 'write' mode then blobfuse upload that file on close, ir-respective of file was modified or not. Setting this flag to true means files will be uploaded only if they were modified after opening. ### Valid authentication setups: - Account Name & Key (`authType Key`) diff --git a/blobfuse-nightly.yml b/blobfuse-nightly.yml index cef3b33b..0806bf84 100755 --- a/blobfuse-nightly.yml +++ b/blobfuse-nightly.yml @@ -70,6 +70,9 @@ jobs: value: "./connection.cfg" - name: BLOBFUSE_STRESS_DIR value: "/home/vsts/work/blob_stress" + - name: DECODE_PERCENTS + value: false + # Each job has set of steps to be done diff --git a/blobfuse/blobfuse.cpp b/blobfuse/blobfuse.cpp index 0c0c8d81..95abc246 100755 --- a/blobfuse/blobfuse.cpp +++ b/blobfuse/blobfuse.cpp @@ -49,7 +49,8 @@ const struct fuse_opt option_spec[] = OPTION("--max-concurrency=%s", concurrency), OPTION("--cache-size-mb=%s", cache_size_mb), OPTION("--empty-dir-check=%s", empty_dir_check), - OPTION("--upload-if-modified=%s", upload_if_modified), + OPTION("--upload-modified-only=%s", upload_if_modified), + OPTION("--cancel-list-on-mount-seconds=%s", cancel_list_on_mount_seconds), OPTION("--high-disk-threshold=%s", high_disk_threshold), OPTION("--low-disk-threshold=%s", low_disk_threshold), OPTION("--version", version), @@ -932,6 +933,12 @@ int read_and_set_arguments(int argc, char *argv[], struct fuse_args *args) config_options.cacheSize = stoi(cache_size) * (unsigned long long)(1024l * 1024l); } + config_options.cancel_list_on_mount_secs = 0; + if (cmd_options.cancel_list_on_mount_seconds != NULL) + { + std::string cancel_list_on_mount_secs(cmd_options.cancel_list_on_mount_seconds); + config_options.cancel_list_on_mount_secs = stoi(cancel_list_on_mount_secs); + } // Make high and low disk threshold percentage a configurable option if (cmd_options.high_disk_threshold != NULL) { @@ -959,7 +966,6 @@ int read_and_set_arguments(int argc, char *argv[], struct fuse_args *args) config_options.low_disk_threshold = LOW_THRESHOLD_VALUE; } syslog(LOG_INFO, "Disk Thresholds : %d - %d", config_options.high_disk_threshold, config_options.low_disk_threshold); - return 0; } diff --git a/blobfuse/directoryapis.cpp b/blobfuse/directoryapis.cpp index 60d85216..fb7833eb 100755 --- a/blobfuse/directoryapis.cpp +++ b/blobfuse/directoryapis.cpp @@ -43,6 +43,12 @@ int azs_mkdir(const char *path, mode_t) int azs_readdir(const char *path, void *buf, fuse_fill_dir_t filler, off_t, struct fuse_file_info *) { AZS_DEBUGLOGV("azs_readdir called with path = %s\n", path); + + if (config_options.cancel_list_on_mount_secs > 0) { + AZS_DEBUGLOGV("azs_readdir blocked by user config for %d more secs\n", config_options.cancel_list_on_mount_secs); + return 0; + } + std::string pathStr(path); if (pathStr.size() > 1) { diff --git a/blobfuse/include/BlobfuseGlobals.h b/blobfuse/include/BlobfuseGlobals.h index 787ea0f2..7ff0fede 100755 --- a/blobfuse/include/BlobfuseGlobals.h +++ b/blobfuse/include/BlobfuseGlobals.h @@ -63,6 +63,7 @@ struct configParams int defaultPermission; int concurrency; unsigned long long cacheSize; + volatile int cancel_list_on_mount_secs; bool emptyDirCheck; bool uploadIfModified; std::string mntPath; @@ -87,6 +88,7 @@ struct cmdlineOptions const char *help; // print blobfuse usage const char *concurrency; // Max Concurrency factor for blob client wrapper (default 40) const char *cache_size_mb; // MAX Size of cache in MBs + const char *cancel_list_on_mount_seconds; // Block the list api call on mount for n seconds const char *empty_dir_check; const char *upload_if_modified; const char *encode_full_file_name; // Encode the '%' symbol in file name diff --git a/blobfuse/src/BlobfuseConstants.cpp b/blobfuse/src/BlobfuseConstants.cpp index 44070f6e..73b035cc 100755 --- a/blobfuse/src/BlobfuseConstants.cpp +++ b/blobfuse/src/BlobfuseConstants.cpp @@ -9,7 +9,7 @@ namespace blobfuse_constants { const int max_concurrency_oauth = 2; const int max_retry_oauth = 5; const int max_concurrency_blob_wrapper = 40; - const int def_concurrency_blob_wrapper = 20; + const int def_concurrency_blob_wrapper = 12; const char* oauth_request_uri = "https://login.microsoftonline.com"; const char* spn_request_path = "oauth2/v2.0/token"; diff --git a/blobfuse/src/BlobfuseGcCache.cpp b/blobfuse/src/BlobfuseGcCache.cpp index 85760e3d..316fe93b 100755 --- a/blobfuse/src/BlobfuseGcCache.cpp +++ b/blobfuse/src/BlobfuseGcCache.cpp @@ -106,7 +106,11 @@ void gc_cache::run_gc_cache() if(is_empty) { //run it every 1 second - usleep(1000); + sleep(1); + if (config_options.cancel_list_on_mount_secs > 0) { + //AZS_DEBUGLOGV("Reducing lst block wait time %d\n", config_options.cancel_list_on_mount_secs); + config_options.cancel_list_on_mount_secs--; + } continue; } @@ -184,7 +188,7 @@ void gc_cache::run_gc_cache() else { // no file was timed out - let's wait a second - usleep(1000); + sleep(1); //check disk space disk_threshold_reached = check_disk_space(); } diff --git a/blobfuse/src/BlockBlobBfsClient.cpp b/blobfuse/src/BlockBlobBfsClient.cpp index 24448a19..ee5f7a75 100755 --- a/blobfuse/src/BlockBlobBfsClient.cpp +++ b/blobfuse/src/BlockBlobBfsClient.cpp @@ -148,7 +148,7 @@ std::shared_ptr BlockBlobBfsClient::authenticate_blob_msi() true, //use_https must be true to use oauth configurations.blobEndpoint); std::shared_ptr blobClient = - std::make_shared(account, max_concurrency_oauth); + std::make_shared(account, configurations.concurrency); errno = 0; return std::make_shared(blobClient); } @@ -192,7 +192,7 @@ std::shared_ptr BlockBlobBfsClient::authenticate_blob_spn() true, //use_https must be true to use oauth configurations.blobEndpoint); std::shared_ptr blobClient = - std::make_shared(account, max_concurrency_oauth); + std::make_shared(account, configurations.concurrency); errno = 0; return std::make_shared(blobClient); } @@ -212,7 +212,7 @@ std::shared_ptr BlockBlobBfsClient::authenticate_blob_spn() void BlockBlobBfsClient::UploadFromFile(const std::string sourcePath, METADATA &metadata) { std::string blobName = sourcePath.substr(configurations.tmpPath.size() + 6 /* there are six characters in "/root/" */); - m_blob_client->upload_file_to_blob(sourcePath, configurations.containerName, blobName, metadata); + m_blob_client->upload_file_to_blob(sourcePath, configurations.containerName, blobName, metadata, configurations.concurrency); // upload_file_to_blob does not return a status or success if the blob succeeded // it does syslog if there was an exception and changes the errno. } @@ -237,7 +237,7 @@ void BlockBlobBfsClient::UploadFromStream(std::istream &sourceStream, const std: ///none long int BlockBlobBfsClient::DownloadToFile(const std::string blobName, const std::string filePath, time_t &last_modified) { - m_blob_client->download_blob_to_file(configurations.containerName, blobName, filePath, last_modified); + m_blob_client->download_blob_to_file(configurations.containerName, blobName, filePath, last_modified, configurations.concurrency); struct stat stbuf; lstat(filePath.c_str(), &stbuf); if (0 == stat(filePath.c_str(), &stbuf)) diff --git a/blobfuse/src/OAuthTokenCredentialManager.cpp b/blobfuse/src/OAuthTokenCredentialManager.cpp index bea7799f..8937d84d 100755 --- a/blobfuse/src/OAuthTokenCredentialManager.cpp +++ b/blobfuse/src/OAuthTokenCredentialManager.cpp @@ -125,7 +125,7 @@ void OAuthTokenCredentialManager::TokenMonitor() // If we fail, explain ourselves and unlock. syslog(LOG_ERR, "OAUTH Token : TokenMonitor : Failed to refresh (%s)\n", ex.what()); syslog(LOG_ERR, "OAUTH Token : TokenMonitor : Retry refreshing the token attempt %d", retry_count++); - usleep(10 * 1000); + usleep(100 * 1000); } } token_mutex.unlock();