From 3086686fcef146c17b71a4905abbf019d81f4b46 Mon Sep 17 00:00:00 2001 From: Ilana Kantorov Date: Mon, 23 Oct 2017 15:07:08 +0000 Subject: [PATCH] Fix code review comments and rename solution --- Gemfile.lock | 6 +- README.md | 31 ++++++--- ...c => fluent-plugin-azuremonitorlog.gemspec | 6 +- ...reactivitylog.rb => in_azuremonitorlog.rb} | 69 ++++++++++--------- test/helper.rb | 2 +- test/plugin/test_in_azureactivitylog.rb | 36 ---------- test/plugin/test_in_azuremonitorlog.rb | 46 +++++++++++++ 7 files changed, 109 insertions(+), 87 deletions(-) rename fluent-plugin-azureactivitylog.gemspec => fluent-plugin-azuremonitorlog.gemspec (78%) rename lib/fluent/plugin/{in_azureactivitylog.rb => in_azuremonitorlog.rb} (71%) delete mode 100644 test/plugin/test_in_azureactivitylog.rb create mode 100644 test/plugin/test_in_azuremonitorlog.rb diff --git a/Gemfile.lock b/Gemfile.lock index 6558583..51efe5c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - fluent-plugin-azureactivitylog (0.0.1) + fluent-plugin-azuremonitorlog (0.0.2) azure_mgmt_monitor (~> 0.11.0) fluentd (>= 0.10.30) @@ -68,9 +68,9 @@ PLATFORMS ruby DEPENDENCIES - fluent-plugin-azureactivitylog! + fluent-plugin-azuremonitorlog! rake (>= 0.9.2) test-unit (>= 3.1.0) BUNDLED WITH - 1.11.2 + 1.15.4 diff --git a/README.md b/README.md index 93d87a9..7a05f2f 100644 --- a/README.md +++ b/README.md @@ -1,37 +1,46 @@ -# fluent-plugin-azureactivitylog, a plugin for [Fluentd](http://fluentd.org) +# fluent-plugin-azuremonitorlog, a plugin for [Fluentd](http://fluentd.org) ## Overview -***Azure Activity log*** input plugin. +***Azure Monitor log*** input plugin. -This plugin gets the activity logs from Azure Monitor API to fluentd. +This plugin gets the monitor activity logs from Azure Monitor API to fluentd. -Installation +## Installation -$ gem install fluent-plugin-azureactivitylog +Install from RubyGems: +``` +$ gem install fluent-plugin-azuremonitorlog +``` + +To use this plugin, you need to have Azure Service Principal.
+Create an Azure Service Principal through [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/create-an-azure-service-principal-azure-cli?toc=%2fazure%2fazure-resource-manager%2ftoc.json) or [Azure portal](https://docs.microsoft.com/en-us/azure/azure-resource-manager/resource-group-create-service-principal-portal). ## Configuration ```config - @type azureactivitylog - tag azureactivitylog + @type azuremonitorlog + tag azuremonitorlog tenant_id [Azure_Tenant_ID] subscription_id [Azure_Subscription_Id] client_id [Azure_Client_Id] client_secret [Azure_Client_Secret] - select_filter [selected fields to query] - event_channels [event channles] (default: Admin, Operation) + select [selected fields to query] + filter [filter the query query] (default: eventChannels eq 'Admin, Operation') interval [interval in seconds] (default: 300) + api_version [api version] (default: 2015-04-01) ``` +Documentation for select and filter can be found [here](https://docs.microsoft.com/en-us/rest/api/monitor/ActivityLogs/List#activitylogs_list_uri_parameters) + ### Example for source config ```config - @type azureactivitylog - tag azureactivitylog + @type azuremonitorlog + tag azuremonitorlog tenant_id [Azure_Tenant_ID] subscription_id [Azure_Subscription_Id] client_id [Azure_Client_Id] diff --git a/fluent-plugin-azureactivitylog.gemspec b/fluent-plugin-azuremonitorlog.gemspec similarity index 78% rename from fluent-plugin-azureactivitylog.gemspec rename to fluent-plugin-azuremonitorlog.gemspec index 61d0a76..acad2d6 100644 --- a/fluent-plugin-azureactivitylog.gemspec +++ b/fluent-plugin-azuremonitorlog.gemspec @@ -3,12 +3,12 @@ lib = File.expand_path('../lib', __FILE__) $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) Gem::Specification.new do |gem| - gem.name = "fluent-plugin-azureactivitylog" + gem.name = "fluent-plugin-azuremonitorlog" gem.version = "0.0.1" gem.authors = ["Ilana Kantorov"] gem.email = ["ilanak@microsoft.com"] - gem.description = %q{Input plugin for Azure Activity logs.} - gem.homepage = "https://github.com/..." + gem.description = %q{Input plugin for Azure Monitor Activity logs.} + gem.homepage = "https://github.com/Ilanak/fluent-plugin-azureamonitorlog" gem.summary = gem.description gem.files = `git ls-files`.split($\) gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) } diff --git a/lib/fluent/plugin/in_azureactivitylog.rb b/lib/fluent/plugin/in_azuremonitorlog.rb similarity index 71% rename from lib/fluent/plugin/in_azureactivitylog.rb rename to lib/fluent/plugin/in_azuremonitorlog.rb index cabd403..e5b67c5 100644 --- a/lib/fluent/plugin/in_azureactivitylog.rb +++ b/lib/fluent/plugin/in_azuremonitorlog.rb @@ -2,8 +2,8 @@ require 'fluent/input' require 'azure_mgmt_monitor' require 'uri' -class Fluent::AzureActivityLogInput < Fluent::Input - Fluent::Plugin.register_input("azureactivitylog", self) +class Fluent::AzureMonitorLogInput < Fluent::Input + Fluent::Plugin.register_input("azuremonitorlog", self) # To support log_level option implemented by Fluentd v0.10.43 unless method_defined?(:log) @@ -21,10 +21,10 @@ class Fluent::AzureActivityLogInput < Fluent::Input config_param :client_id, :string, :default => nil config_param :client_secret, :string, :default => nil, :secret => true - config_param :select_filter, :string, :default => nil - config_param :event_channels, :string, :default => "Admin, Operation" - config_param :interval, :integer, :default => 300 - + config_param :select, :string, :default => nil + config_param :filter, :string, :default => "eventChannels eq 'Admin, Operation'" + config_param :interval, :integer,:default => 300 + config_param :api_version, :string, :default => '2015-04-01' def initialize super end @@ -49,11 +49,29 @@ class Fluent::AzureActivityLogInput < Fluent::Input @watcher.join end + def set_query_options(filter, custom_headers) + fail ArgumentError, 'path is nil' if @client.subscription_id.nil? + + request_headers = {} + + # Set Headers + request_headers['x-ms-client-request-id'] = SecureRandom.uuid + request_headers['accept-language'] = @client.accept_language unless @client.accept_language.nil? + + { + middlewares: [[MsRest::RetryPolicyMiddleware, times: 3, retry: 0.02], [:cookie_jar]], + path_params: {'subscriptionId' => @client.subscription_id}, + query_params: {'api-version' => @api_version, '$filter' => filter, '$select' => @select}, + headers: request_headers.merge(custom_headers || {}), + base_url: @client.base_url + } + end + private def watch while true - log.debug "azure activitylog: watch thread starting" + log.debug "azure monitorlog: watch thread starting" output sleep @interval end @@ -66,40 +84,25 @@ class Fluent::AzureActivityLogInput < Fluent::Input log.debug "start time: #{start_time}, end time: #{end_time}" filter = "eventTimestamp ge '#{start_time}' and eventTimestamp le '#{end_time}'" - if !@event_channels.empty? - filter += " and eventChannels eq '#{@event_channels}'" + if !@filter.empty? + filter += " and #{@filter}" end - activity_logs_promise = get_activity_log_async(filter) - activity_logs = activity_logs_promise.value! + monitor_logs_promise = get_monitor_log_async(filter) + monitor_logs = monitor_logs_promise.value! - if activity_logs.body.values[0].any? - activity_logs.body.values[0].each {|val| - router.emit(@tag, Time.now.to_i, val.to_json) + if !monitor_logs.body['value'].nil? and monitor_logs.body['value'].any? + monitor_logs.body['value'].each {|val| + router.emit(@tag, Time.now.to_i, val) } else log.debug "empty" end end - def get_activity_log_async(filter = nil, custom_headers = nil) - fail ArgumentError, 'path is nil' if @client.subscription_id.nil? - api_version = '2015-04-01' - - request_headers = {} - - # Set Headers - request_headers['x-ms-client-request-id'] = SecureRandom.uuid - request_headers['accept-language'] = @client.accept_language unless @client.accept_language.nil? + def get_monitor_log_async(filter = nil, custom_headers = nil) + options = set_query_options(filter, custom_headers) path_template = '/subscriptions/{subscriptionId}/providers/microsoft.insights/eventtypes/management/values' - - options = { - middlewares: [[MsRest::RetryPolicyMiddleware, times: 3, retry: 0.02], [:cookie_jar]], - path_params: {'subscriptionId' => @client.subscription_id}, - query_params: {'api-version' => api_version, '$filter' => filter, '$select' => @select_filter}, - headers: request_headers.merge(custom_headers || {}), - base_url: @client.base_url - } promise = @client.make_request_async(:get, path_template, options) promise = promise.then do |result| @@ -108,7 +111,7 @@ class Fluent::AzureActivityLogInput < Fluent::Input response_content = http_response.body unless status_code == 200 error_model = JSON.load(response_content) - fail MsRestAzure::AzureOperationError.new(result.request, http_response, error_model) + log.error(error_model['error']['message']) end result.request_id = http_response['x-ms-request-id'] unless http_response['x-ms-request-id'].nil? @@ -117,7 +120,7 @@ class Fluent::AzureActivityLogInput < Fluent::Input begin result.body = response_content.to_s.empty? ? nil : JSON.load(response_content) rescue Exception => e - fail MsRest::DeserializationError.new('Error occurred in parsing the response', e.message, e.backtrace, result) + log.error("Error occurred in parsing the response") end end diff --git a/test/helper.rb b/test/helper.rb index 256375e..ffd5272 100644 --- a/test/helper.rb +++ b/test/helper.rb @@ -24,7 +24,7 @@ unless ENV.has_key?("VERBOSE") $log = nulllogger end -require "fluent/plugin/in_azureactivitylog" +require "fluent/plugin/in_azuremonitorlog" class Test::Unit::TestCase end diff --git a/test/plugin/test_in_azureactivitylog.rb b/test/plugin/test_in_azureactivitylog.rb deleted file mode 100644 index ca76980..0000000 --- a/test/plugin/test_in_azureactivitylog.rb +++ /dev/null @@ -1,36 +0,0 @@ -require 'helper' - -class AzureActivityLogInputTest < Test::Unit::TestCase - def setup - Fluent::Test.setup - end - - ### for activity log - CONFIG_ACTIVITY_LOG = %[ - tag azureactivitylog - tenant_id test_tenant_id - subscription_id test_subscription_id - client_id test_client_id - client_secret test_client_secret - select_filter eventName,id,resourceGroupName,resourceProviderName,operationName,status,eventTimestamp,correlationId - event_channels test_event_channels - interval 300 - ] - - def create_driver_activity_log(conf = CONFIG_ACTIVITY_LOG) - Fluent::Test::InputTestDriver.new(Fluent::AzureActivityLogInput).configure(conf) - end - - def test_configure_activity_log - d = create_driver_activity_log - assert_equal 'azureactivitylog', d.instance.tag - assert_equal 'test_tenant_id', d.instance.tenant_id - assert_equal 'test_subscription_id', d.instance.subscription_id - assert_equal 'test_client_id', d.instance.client_id - assert_equal 'test_client_secret', d.instance.client_secret - assert_equal 'eventName,id,resourceGroupName,resourceProviderName,operationName,status,eventTimestamp,correlationId', d.instance.select_filter - assert_equal 'test_event_channels', d.instance.event_channels - assert_equal 300, d.instance.interval - end - -end diff --git a/test/plugin/test_in_azuremonitorlog.rb b/test/plugin/test_in_azuremonitorlog.rb new file mode 100644 index 0000000..e7b2ca3 --- /dev/null +++ b/test/plugin/test_in_azuremonitorlog.rb @@ -0,0 +1,46 @@ +require 'helper' + +class AzureMonitorLogInputTest < Test::Unit::TestCase + def setup + Fluent::Test.setup + end + + ### for monitor log + CONFIG_MONITOR_LOG = %[ + tag azuremonitorlog + tenant_id test_tenant_id + subscription_id test_subscription_id + client_id test_client_id + client_secret test_client_secret + select eventName,id,resourceGroupName,resourceProviderName,operationName,status,eventTimestamp,correlationId + filter eventChannels eq 'Admin, Operation' + interval 300 + api_version 2015-04-01 + ] + + def create_driver_monitor_log(conf = CONFIG_MONITOR_LOG) + Fluent::Test::InputTestDriver.new(Fluent::AzureMonitorLogInput).configure(conf) + end + + def test_configure_monitor_log + d = create_driver_monitor_log + assert_equal 'azuremonitorlog', d.instance.tag + assert_equal 'test_tenant_id', d.instance.tenant_id + assert_equal 'test_subscription_id', d.instance.subscription_id + assert_equal 'test_client_id', d.instance.client_id + assert_equal 'test_client_secret', d.instance.client_secret + assert_equal 'eventName,id,resourceGroupName,resourceProviderName,operationName,status,eventTimestamp,correlationId', d.instance.select + assert_equal 'eventChannels eq \'Admin, Operation\'', d.instance.filter + assert_equal 300, d.instance.interval + assert_equal '2015-04-01', d.instance.api_version + end + + def test_set_query_options + d = create_driver_monitor_log + query_options = d.instance.set_query_options(d.instance.filter, {}) + assert_equal '2015-04-01', query_options[:query_params]['api-version'] + assert_equal 'eventChannels eq \'Admin, Operation\'', query_options[:query_params]['$filter'] + assert_equal 'eventName,id,resourceGroupName,resourceProviderName,operationName,status,eventTimestamp,correlationId', query_options[:query_params]['$select'] + end + +end