Package and run td-agent config for metrics

.gitignore

@@ -286,3 +286,4 @@ __pycache__/
 *.btm.cs
 *.odx.cs
 *.xsd.cs
+scripts/td-agent-config

FluentD/td-agent.cloudera.conf

@@ -27,7 +27,7 @@
   utc
   <buffer tag,time>
     @type file
-    path /var/log/td-agent/azurestorage/clouderametrics
+    path /var/log/td-agent/metrics-logs/clouderametrics
     timekey 120 # partitioning time in seconds
     timekey_wait 1m
     timekey_use_utc true # use utc

FluentD/td-agent.metrics.conf

@@ -44,7 +44,14 @@
   @type record_transformer
   <record>
     hostname ${hostname}
-    env "#{ENV["env"]}"
+    environment "#{ENV["AZ_ENVIRONMENT"]}"
+    ait "#{ENV["AZ_AIT"]}"
+    dns_ip_address "#{ENV["AZ_DNS_IP_ADDRESS"]}"
+    runid "#{ENV["AZ_RUNID"]}"
+    archive "#{ENV["AZ_ARCHIVE"]}"
+    creatorid "#{ENV["AZ_CREATORID"]}"
+    createddate "#{ENV["AZ_CREATEDDATE"]}"
+    uniquename "#{ENV["AZ_UNIQUENAME"]}"
   </record>
 </filter>
 
@@ -64,9 +71,9 @@
   utc
   <buffer tag,time>
     @type file
-    path /var/log/td-agent/azurestorage/perfcounters
+    path /var/log/td-agent/metrics-logs/perfcounters
     timekey 120 # partitioning time in seconds
     timekey_wait 1m
     timekey_use_utc true # use utc
   </buffer>
-</match>
+</match>

README.md

@@ -47,7 +47,7 @@ ansible all -m ping -u <username> -i hosts --ask-pass
 ```
 3. Run Ansible playbook
 ```
-ansible-playbook playbook.yml -i ansible_inventory --ask-pass --ask-pass --ask-become-pass
+ansible-playbook playbook.setup.yml -i ansible_inventory --ask-pass --ask-pass --ask-become-pass
 ```
 
 # Splunk

ansible/playbook.setup.yml

@@ -1,6 +1,6 @@
 ---
-- hosts: cdh_servers
   remote_user: <admin user>
+- hosts: scm_server
   become: true
   become_method: sudo
   tasks:

scripts/configure-td-agent.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+source /opt/rh/python27/enable
+
+# Set the environment variables for the td-agent service
+python azure_tags_env_variables.py
+cat AzureTagsEnvVariables > /etc/sysconfig/td-agent
+
+# Stop the td-agent
+/etc/init.d/td-agent stop
+
+# Create a backup of the td-agent.conf
+mv -f /etc/td-agent/td-agent.conf /etc/td-agent/td-agent.conf.bak
+
+# Append custom configs to existing td-agent.conf
+if [[ $1 = "cm" ]]; then
+    echo "Appending Clouder Manager and Metrics td-agent config..."
+    cp in_clouderametrics.rb /etc/td-agent/plugin
+    cat td-agent.cloudera.conf td-agent.metrics.conf /etc/td-agent/td-agent.conf.bak > /etc/td-agent/td-agent.conf
+else
+    echo "Appending Metrics td-agent config..."
+    cat td-agent.metrics.conf /etc/td-agent/td-agent.conf.bak > /etc/td-agent/td-agent.conf
+fi
+
+# Restart the td-agent
+/etc/init.d/td-agent restart

scripts/package-tar-file.sh

@@ -0,0 +1,5 @@
+cp ../FluentD/td-agent.metrics.conf td-agent-config/
+cp ../FluentD/td-agent.cloudera.conf td-agent-config/
+cp configure-td-agent.sh td-agent-config/
+cp azure_tags_env_variables.py td-agent-config/
+tar -cvzf td-agent.tar.gz td-agent-config 

scripts/td-agent-config/README.md

@@ -0,0 +1,10 @@
+This folder is temporary and is used by package-tar-file.sh to tar the td-agent conf files
+
+##### Apply Configuration Steps
+1. Download the tar onto a VM
+2. tar -xvf td-agent.tar.gz
+3. pushd .
+4. cd td-agent-config
+5. chmod 744 configure-td-agent.sh
+6. ./configure-td-agent.sh or ./configure-td-agent.sh cm
+7. popd