From 05d68fe42e0448bc49935ecbda9b7a644175d275 Mon Sep 17 00:00:00 2001 From: ilanaLinux Date: Tue, 17 Oct 2017 14:22:05 +0100 Subject: [PATCH] initial commit --- Gemfile | 4 + Gemfile.fluentd.0.12 | 4 + Gemfile.lock | 76 +++++++++++++ LICENSE | 35 +++--- README.md | 47 ++++++++- Rakefile | 11 ++ fluent-plugin-azureactivitylog.gemspec | 22 ++++ lib/fluent/plugin/in_azureactivitylog.rb | 129 +++++++++++++++++++++++ test/helper.rb | 31 ++++++ test/plugin/test_in_azureactivitylog.rb | 36 +++++++ 10 files changed, 377 insertions(+), 18 deletions(-) create mode 100644 Gemfile create mode 100644 Gemfile.fluentd.0.12 create mode 100644 Gemfile.lock create mode 100644 Rakefile create mode 100644 fluent-plugin-azureactivitylog.gemspec create mode 100644 lib/fluent/plugin/in_azureactivitylog.rb create mode 100644 test/helper.rb create mode 100644 test/plugin/test_in_azureactivitylog.rb diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..82b87f8 --- /dev/null +++ b/Gemfile @@ -0,0 +1,4 @@ +source 'https://rubygems.org' + +# Specify your gem's dependencies in fluent-plugin-azuremonitor.gemspec +gemspec diff --git a/Gemfile.fluentd.0.12 b/Gemfile.fluentd.0.12 new file mode 100644 index 0000000..a13628e --- /dev/null +++ b/Gemfile.fluentd.0.12 @@ -0,0 +1,4 @@ +source "http://rubygems.org" + +gem 'fluentd', '~> 0.12.0' +gemspec diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 0000000..6558583 --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,76 @@ +PATH + remote: . + specs: + fluent-plugin-azureactivitylog (0.0.1) + azure_mgmt_monitor (~> 0.11.0) + fluentd (>= 0.10.30) + +GEM + remote: https://rubygems.org/ + specs: + azure_mgmt_monitor (0.11.0) + ms_rest_azure (~> 0.8.0) + concurrent-ruby (1.0.5) + cool.io (1.5.1) + domain_name (0.5.20170404) + unf (>= 0.0.5, < 1.0.0) + faraday (0.13.1) + multipart-post (>= 1.2, < 3) + faraday-cookie_jar (0.0.6) + faraday (>= 0.7.4) + http-cookie (~> 1.0.0) + fluentd (0.14.21) + cool.io (>= 1.4.5, < 2.0.0) + http_parser.rb (>= 0.5.1, < 0.7.0) + msgpack (>= 0.7.0, < 2.0.0) + ruby_dig (~> 0.0.2) + serverengine (>= 2.0.4, < 3.0.0) + sigdump (~> 0.2.2) + strptime (~> 0.1.7) + tzinfo (~> 1.0) + tzinfo-data (~> 1.0) + yajl-ruby (~> 1.0) + http-cookie (1.0.3) + domain_name (~> 0.5) + http_parser.rb (0.6.0) + ms_rest (0.7.1) + concurrent-ruby (~> 1.0) + faraday (~> 0.9) + timeliness (~> 0.3) + ms_rest_azure (0.8.2) + concurrent-ruby (~> 1.0) + faraday (~> 0.9) + faraday-cookie_jar (~> 0.0.6) + ms_rest (~> 0.7.0) + msgpack (1.1.0) + multipart-post (2.0.0) + power_assert (1.1.0) + rake (12.0.0) + ruby_dig (0.0.2) + serverengine (2.0.5) + sigdump (~> 0.2.2) + sigdump (0.2.4) + strptime (0.1.9) + test-unit (3.2.5) + power_assert + thread_safe (0.3.6) + timeliness (0.3.8) + tzinfo (1.2.3) + thread_safe (~> 0.1) + tzinfo-data (1.2017.2) + tzinfo (>= 1.0.0) + unf (0.1.4) + unf_ext + unf_ext (0.0.7.4) + yajl-ruby (1.3.0) + +PLATFORMS + ruby + +DEPENDENCIES + fluent-plugin-azureactivitylog! + rake (>= 0.9.2) + test-unit (>= 3.1.0) + +BUNDLED WITH + 1.11.2 diff --git a/LICENSE b/LICENSE index 91f2830..6be34e5 100644 --- a/LICENSE +++ b/LICENSE @@ -1,21 +1,22 @@ +Copyright (c) 2012 Yusuke Nomura + MIT License -Copyright (c) 2017 Ilana Kantorov +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md index 882176f..99c5fcc 100644 --- a/README.md +++ b/README.md @@ -1 +1,46 @@ -# fluent-plugin-azureactivitylog \ No newline at end of file +# fluent-plugin-azureactivitylog, a plugin for [Fluentd](http://fluentd.org) +## Overview + +***Azure Activity log*** input plugin. + +This plugin is simple, it gets the activity logs from Azure Monitor API to fluentd.. + +## Configuration + +```config + + @type azureactivitylog + tag azureactivitylog + tenant_id [Azure_Tenant_ID] + subscription_id [Azure_Subscription_Id] + client_id [Azure_Client_Id] + client_secret [Azure_Client_Secret] + + select_filter [selected fields to query] + event_channels [event channles] (default: Admin, Operation) + interval [interval in seconds] (default: 300) + +``` + +### Example for source config + +```config + + @type azureactivitylog + tag azureactivitylog + tenant_id [Azure_Tenant_ID] + subscription_id [Azure_Subscription_Id] + client_id [Azure_Client_Id] + client_secret [Azure_Client_Secret] + select_filter eventName,id,resourceGroupName,resourceProviderName,operationName,status,eventTimestamp,correlationId,submissionTimestamp,level + + +``` + +## Contributing + +1. Fork it +2. Create your feature branch (`git checkout -b my-new-feature`) +3. Commit your changes (`git commit -am 'Added some feature'`) +4. Push to the branch (`git push origin my-new-feature`) +5. Create new Pull Request diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000..cd78b2d --- /dev/null +++ b/Rakefile @@ -0,0 +1,11 @@ +#!/usr/bin/env rake +require "bundler/gem_tasks" + +require "rake/testtask" +Rake::TestTask.new(:test) do |test| + test.libs << 'lib' << 'test' + test.pattern = 'test/**/test_*.rb' + test.verbose = true +end + +task :default => :test diff --git a/fluent-plugin-azureactivitylog.gemspec b/fluent-plugin-azureactivitylog.gemspec new file mode 100644 index 0000000..61d0a76 --- /dev/null +++ b/fluent-plugin-azureactivitylog.gemspec @@ -0,0 +1,22 @@ +# -*- encoding: utf-8 -*- +lib = File.expand_path('../lib', __FILE__) +$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) + +Gem::Specification.new do |gem| + gem.name = "fluent-plugin-azureactivitylog" + gem.version = "0.0.1" + gem.authors = ["Ilana Kantorov"] + gem.email = ["ilanak@microsoft.com"] + gem.description = %q{Input plugin for Azure Activity logs.} + gem.homepage = "https://github.com/..." + gem.summary = gem.description + gem.files = `git ls-files`.split($\) + gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) } + gem.test_files = gem.files.grep(%r{^(test|spec|features)/}) + gem.require_paths = ["lib"] + gem.add_dependency "fluentd", ">= 0.10.30" + gem.add_dependency "azure_mgmt_monitor", "~> 0.11.0" + gem.add_development_dependency "rake", ">= 0.9.2" + gem.add_development_dependency "test-unit", ">= 3.1.0" + gem.license = 'MIT' +end diff --git a/lib/fluent/plugin/in_azureactivitylog.rb b/lib/fluent/plugin/in_azureactivitylog.rb new file mode 100644 index 0000000..cabd403 --- /dev/null +++ b/lib/fluent/plugin/in_azureactivitylog.rb @@ -0,0 +1,129 @@ +require 'fluent/input' +require 'azure_mgmt_monitor' +require 'uri' + +class Fluent::AzureActivityLogInput < Fluent::Input + Fluent::Plugin.register_input("azureactivitylog", self) + + # To support log_level option implemented by Fluentd v0.10.43 + unless method_defined?(:log) + define_method("log") { $log } + end + + # Define `router` method of v0.12 to support v0.10 or earlier + unless method_defined?(:router) + define_method("router") { Fluent::Engine } + end + + config_param :tag, :string + config_param :tenant_id, :string, :default => nil + config_param :subscription_id, :string, :default => nil + config_param :client_id, :string, :default => nil + config_param :client_secret, :string, :default => nil, :secret => true + + config_param :select_filter, :string, :default => nil + config_param :event_channels, :string, :default => "Admin, Operation" + config_param :interval, :integer, :default => 300 + + def initialize + super + end + + def configure(conf) + super + + provider = MsRestAzure::ApplicationTokenProvider.new(@tenant_id, @client_id, @client_secret) + credentials = MsRest::TokenCredentials.new(provider) + @client = Azure::ARM::Monitor::MonitorManagementClient.new(credentials); + @client.subscription_id = @subscription_id + end + + def start + super + @watcher = Thread.new(&method(:watch)) + end + + def shutdown + super + @watcher.terminate + @watcher.join + end + + private + + def watch + while true + log.debug "azure activitylog: watch thread starting" + output + sleep @interval + end + end + + def output + start_time = Time.now - @interval + end_time = Time.now + + log.debug "start time: #{start_time}, end time: #{end_time}" + filter = "eventTimestamp ge '#{start_time}' and eventTimestamp le '#{end_time}'" + + if !@event_channels.empty? + filter += " and eventChannels eq '#{@event_channels}'" + end + + activity_logs_promise = get_activity_log_async(filter) + activity_logs = activity_logs_promise.value! + + if activity_logs.body.values[0].any? + activity_logs.body.values[0].each {|val| + router.emit(@tag, Time.now.to_i, val.to_json) + } + else + log.debug "empty" + end + end + + def get_activity_log_async(filter = nil, custom_headers = nil) + fail ArgumentError, 'path is nil' if @client.subscription_id.nil? + api_version = '2015-04-01' + + request_headers = {} + + # Set Headers + request_headers['x-ms-client-request-id'] = SecureRandom.uuid + request_headers['accept-language'] = @client.accept_language unless @client.accept_language.nil? + path_template = '/subscriptions/{subscriptionId}/providers/microsoft.insights/eventtypes/management/values' + + options = { + middlewares: [[MsRest::RetryPolicyMiddleware, times: 3, retry: 0.02], [:cookie_jar]], + path_params: {'subscriptionId' => @client.subscription_id}, + query_params: {'api-version' => api_version, '$filter' => filter, '$select' => @select_filter}, + headers: request_headers.merge(custom_headers || {}), + base_url: @client.base_url + } + promise = @client.make_request_async(:get, path_template, options) + + promise = promise.then do |result| + http_response = result.response + status_code = http_response.status + response_content = http_response.body + unless status_code == 200 + error_model = JSON.load(response_content) + fail MsRestAzure::AzureOperationError.new(result.request, http_response, error_model) + end + + result.request_id = http_response['x-ms-request-id'] unless http_response['x-ms-request-id'].nil? + # Deserialize Response + if status_code == 200 + begin + result.body = response_content.to_s.empty? ? nil : JSON.load(response_content) + rescue Exception => e + fail MsRest::DeserializationError.new('Error occurred in parsing the response', e.message, e.backtrace, result) + end + end + + result + end + + promise.execute + end +end diff --git a/test/helper.rb b/test/helper.rb new file mode 100644 index 0000000..256375e --- /dev/null +++ b/test/helper.rb @@ -0,0 +1,31 @@ +require 'rubygems' +require 'bundler' +require 'fluent/input' + +begin + Bundler.setup(:default, :development) +rescue Bundler::BundlerError => e + $stderr.puts e.message + $stderr.puts "Run `bundle install` to install missing gems" + exit e.status_code +end +require "test/unit" + +$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), "..", "lib")) +$LOAD_PATH.unshift(File.dirname(__FILE__)) +require "fluent/test" +unless ENV.has_key?("VERBOSE") + nulllogger = Object.new + nulllogger.instance_eval {|obj| + def method_missing(method, *args) + #pass + end + } + $log = nulllogger +end + +require "fluent/plugin/in_azureactivitylog" + +class Test::Unit::TestCase +end + diff --git a/test/plugin/test_in_azureactivitylog.rb b/test/plugin/test_in_azureactivitylog.rb new file mode 100644 index 0000000..e89b1ce --- /dev/null +++ b/test/plugin/test_in_azureactivitylog.rb @@ -0,0 +1,36 @@ +require 'helper' + +class AzureActivityLogInputTest < Test::Unit::TestCase + def setup + Fluent::Test.setup + end + + ### for activity log + CONFIG_ACTIVITY_LOG = %[ + tag azureactivitylog + tenant_id test_tenant_id + subscription_id test_subscription_id + client_id test_client_id + client_secret test_client_secret + select_filter eventName,id,resourceGroupName,resourceProviderName,operationName,status,eventTimestamp,correlationId + event_channels test_event_channels + interval 300 + ] + + def create_driver_activity_log(conf = CONFIG_ACTIVITY_LOG) + Fluent::Test::InputTestDriver.new(Fluent::AzureActivityLogInput).configure(conf) + end + + def test_configure_activity_log + d = create_driver_activity_log + assert_equal 'azureactivitylog', d.instance.tag + assert_equal 'test_tenant_id', d.instance.tenant_id + assert_equal 'test_subscription_id', d.instance.subscription_id + assert_equal 'test_client_id', d.instance.client_id + assert_equal 'test_client_secret', d.instance.client_secret + assert_equal 'eventName,id,resourceGroupName,resourceProviderName,operationName,status,eventTimestamp,correlationId', d.instance.select_filter + assert_equal 'test_event_channels', d.instance.event_channels + assert_equal '300', d.instance.interval + end + +end