diff --git a/.gitignore b/.gitignore index 940794e..a9f7eb7 100644 --- a/.gitignore +++ b/.gitignore @@ -286,3 +286,4 @@ __pycache__/ *.btm.cs *.odx.cs *.xsd.cs +scripts/td-agent-config diff --git a/FluentD/td-agent.cloudera.conf b/FluentD/td-agent.cloudera.conf index 04aa17f..ce54842 100644 --- a/FluentD/td-agent.cloudera.conf +++ b/FluentD/td-agent.cloudera.conf @@ -27,7 +27,7 @@ utc @type file - path /var/log/td-agent/azurestorage/clouderametrics + path /var/log/td-agent/metrics-logs/clouderametrics timekey 120 # partitioning time in seconds timekey_wait 1m timekey_use_utc true # use utc diff --git a/FluentD/td-agent.metrics.conf b/FluentD/td-agent.metrics.conf index 85b4638..b49be3f 100644 --- a/FluentD/td-agent.metrics.conf +++ b/FluentD/td-agent.metrics.conf @@ -44,7 +44,14 @@ @type record_transformer hostname ${hostname} - env "#{ENV["env"]}" + environment "#{ENV["AZ_ENVIRONMENT"]}" + ait "#{ENV["AZ_AIT"]}" + dns_ip_address "#{ENV["AZ_DNS_IP_ADDRESS"]}" + runid "#{ENV["AZ_RUNID"]}" + archive "#{ENV["AZ_ARCHIVE"]}" + creatorid "#{ENV["AZ_CREATORID"]}" + createddate "#{ENV["AZ_CREATEDDATE"]}" + uniquename "#{ENV["AZ_UNIQUENAME"]}" @@ -64,9 +71,9 @@ utc @type file - path /var/log/td-agent/azurestorage/perfcounters + path /var/log/td-agent/metrics-logs/perfcounters timekey 120 # partitioning time in seconds timekey_wait 1m timekey_use_utc true # use utc - \ No newline at end of file + diff --git a/README.md b/README.md index 14e8475..f58de8d 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ ansible all -m ping -u -i hosts --ask-pass ``` 3. Run Ansible playbook ``` -ansible-playbook playbook.yml -i ansible_inventory --ask-pass --ask-pass --ask-become-pass +ansible-playbook playbook.setup.yml -i ansible_inventory --ask-pass --ask-pass --ask-become-pass ``` # Splunk diff --git a/ansible/playbook.setup.yml b/ansible/playbook.setup.yml index f1f9578..b1e1de9 100644 --- a/ansible/playbook.setup.yml +++ b/ansible/playbook.setup.yml @@ -1,6 +1,6 @@ --- -- hosts: cdh_servers remote_user: +- hosts: scm_server become: true become_method: sudo tasks: diff --git a/scripts/configure-td-agent.sh b/scripts/configure-td-agent.sh new file mode 100644 index 0000000..7f7f355 --- /dev/null +++ b/scripts/configure-td-agent.sh @@ -0,0 +1,25 @@ +#!/bin/bash +source /opt/rh/python27/enable + +# Set the environment variables for the td-agent service +python azure_tags_env_variables.py +cat AzureTagsEnvVariables > /etc/sysconfig/td-agent + +# Stop the td-agent +/etc/init.d/td-agent stop + +# Create a backup of the td-agent.conf +mv -f /etc/td-agent/td-agent.conf /etc/td-agent/td-agent.conf.bak + +# Append custom configs to existing td-agent.conf +if [[ $1 = "cm" ]]; then + echo "Appending Clouder Manager and Metrics td-agent config..." + cp in_clouderametrics.rb /etc/td-agent/plugin + cat td-agent.cloudera.conf td-agent.metrics.conf /etc/td-agent/td-agent.conf.bak > /etc/td-agent/td-agent.conf +else + echo "Appending Metrics td-agent config..." + cat td-agent.metrics.conf /etc/td-agent/td-agent.conf.bak > /etc/td-agent/td-agent.conf +fi + +# Restart the td-agent +/etc/init.d/td-agent restart \ No newline at end of file diff --git a/scripts/package-tar-file.sh b/scripts/package-tar-file.sh new file mode 100755 index 0000000..2937e13 --- /dev/null +++ b/scripts/package-tar-file.sh @@ -0,0 +1,5 @@ +cp ../FluentD/td-agent.metrics.conf td-agent-config/ +cp ../FluentD/td-agent.cloudera.conf td-agent-config/ +cp configure-td-agent.sh td-agent-config/ +cp azure_tags_env_variables.py td-agent-config/ +tar -cvzf td-agent.tar.gz td-agent-config \ No newline at end of file diff --git a/scripts/td-agent-config/README.md b/scripts/td-agent-config/README.md new file mode 100644 index 0000000..df92b0b --- /dev/null +++ b/scripts/td-agent-config/README.md @@ -0,0 +1,10 @@ +This folder is temporary and is used by package-tar-file.sh to tar the td-agent conf files + +##### Apply Configuration Steps +1. Download the tar onto a VM +2. tar -xvf td-agent.tar.gz +3. pushd . +4. cd td-agent-config +5. chmod 744 configure-td-agent.sh +6. ./configure-td-agent.sh or ./configure-td-agent.sh cm +7. popd \ No newline at end of file