From c81836c98be71d1a7c4b30159c8f92ece91fe07a Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 23 Jun 2016 07:00:09 -0700 Subject: [PATCH 1/7] Convert consul_template::watch to config_hash --- manifests/watch.pp | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/manifests/watch.pp b/manifests/watch.pp index 779c455..e92b90a 100644 --- a/manifests/watch.pp +++ b/manifests/watch.pp @@ -4,24 +4,31 @@ # This is a single instance of a configuration file to watch # for changes in Consul and update the local file define consul_template::watch ( - $command, - $destination, - $source = undef, + $config_hash = {}, $template = undef, $template_vars = {}, ) { include consul_template - if $template == undef and $source == undef { - err ('Specify either template or source parameter for consul_template::watch') + if $template == undef and $config_hash['source'] == undef { + err ('Specify either template parameter or config_hash["source"] for consul_template::watch') } - if $template != undef and $source != undef { - err ('Specify either template or source parameter for consul_template::watch - but not both') + if $template != undef and $config_hash['source'] != undef { + err ('Specify either template parameter or config_hash["source"] for consul_template::watch - but not both') } - if $template != undef { - file { "${consul_template::config_dir}/${name}.ctmpl": + if $template == undef { + # source is specified in config_hash + $config_source = {} + $frag_name = $config_hash['source'] + } else { + # source is specified as a template + $source = "${consul_template::config_dir}/${name}.ctmpl" + $config_source = { + source => $source, + } + file { $source: ensure => present, owner => $consul_template::user, group => $consul_template::group, @@ -30,19 +37,14 @@ define consul_template::watch ( before => Concat::Fragment["${name}.ctmpl"], notify => Service['consul-template'], } + $frag_name = $source } - if $source != undef { - $source_name = $source - $frag_name = $source - } else { - $source_name = "${consul_template::config_dir}/${name}.ctmpl" - $frag_name = "${name}.ctmpl" - } - + $config_hash_real = deep_merge($config_hash, $config_source) + $content = consul_sorted_json($config_hash_real, $consul::pretty_config, $consul::pretty_config_indent) concat::fragment { $frag_name: target => 'consul-template/config.json', - content => "template {\n source = \"${source_name}\"\n destination = \"${destination}\"\n command = \"${command}\"\n}\n\n", + content => "template ${content}\n\n", order => '10', notify => Service['consul-template'] } From 9d6b04246ed517d827f374f86c3fa49606d0fa06 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 23 Jun 2016 22:11:17 -0700 Subject: [PATCH 2/7] Reworked to fully support config_hash style configuration --- README.markdown | 32 ++++++++------- manifests/config.pp | 94 ++++++++------------------------------------- manifests/init.pp | 30 +++++---------- manifests/params.pp | 1 - manifests/watch.pp | 12 +++--- 5 files changed, 51 insertions(+), 118 deletions(-) diff --git a/README.markdown b/README.markdown index 8c0c16f..9958fa9 100644 --- a/README.markdown +++ b/README.markdown @@ -1,5 +1,13 @@ #consul_template for Puppet +### NOTICE: + +This fork is not backwards compatible with its upstream. The configuration +mechanism has been modified to be driven by config_hash/config_defaults instead +of individual parameters to provide access to the full set of configuration +options and avoid the need to make changes here as consul-template options +evolve. + ##Installation ###What This Module Affects @@ -33,14 +41,9 @@ set to 'package', its installed using the system package manager. - `group` Default: root. - `manage_user` Default: false. Module handles creating the user. - `manage_group` Default: false. Module handles creating the group. -- `consul_host` Default: localhost. Hostanme of consul agent to query -- `consul_port` Default: 8500. Port number the API is running on -- `consul_token` Default: ''. ACL token to use when querying consul -- `consul_retry` Default: 10s. Time in seconds to wait before retrying consul requests -- `consul_wait` Default: undef. Min:Max time to wait before consul-template renders a new template to disk and triggers refresh. Specified in the format min:max according to [Go time duration format](http://golang.org/pkg/time/#ParseDuration) -- `consul_max_stale` Default: undef. The maximum staleness of a query. If specified, Consul will distribute work among all servers instead of just the leader. - `init_style` Init style to use for consul-template service. -- `log_level` Default: info. Logging level to use for consul-template service. Can be 'debug', 'warn', 'err', 'info' +- `config_hash` Default: {}. Consul-template configuration options +- `config_defaults` Default: {}. Consul-template configuration option defaults. @@ -55,14 +58,15 @@ Or to specify parameters: ```puppet class { 'consul_template': service_enable => false - log_level => 'debug', init_style => 'upstart', - consul_wait => '5s:30s', - consul_max_stale => '1s' + config_hash => { + log_level => 'debug', + wait => '5s:30s', + max_stale => '1s' + } } ``` - ## Watch files To declare a file that you wish to populate from Consul key-values, you use the @@ -76,8 +80,10 @@ consul_template::watch { 'common': 'var1' => 'foo', 'var2' => 'bar', }, - destination => '/tmp/common.json', - command => 'true', + config_hash => { + destination => '/tmp/common.json', + command => 'true', + }, } ``` diff --git a/manifests/config.pp b/manifests/config.pp index 0ae9f77..c9d0df4 100644 --- a/manifests/config.pp +++ b/manifests/config.pp @@ -3,89 +3,25 @@ # This class is called from consul_template for service config. # class consul_template::config ( - $consul_host, - $consul_port, - $consul_token, - $consul_retry, - $purge = true, + $config_hash = {}, + $purge = true, ) { + # Using our parent module's pretty_config & pretty_config_indent just because + $content = consul_sorted_json($config_hash, $consul::pretty_config, $consul::pretty_config_indent)[0,-1] - concat::fragment { 'header': - target => 'consul-template/config.json', - content => inline_template("consul = \"<%= @consul_host %>:<%= @consul_port %>\"\ntoken = \"<%= @consul_token %>\"\nretry = \"<%= @consul_retry %>\"\n\n"), - order => '00', + $config_file = "${consul_template::config_dir}/config/config.json" + concat::fragment { 'consul-service-pre': + target => $config_file, + content => "${content},\n \"template\": [\n", + order => '1', } - # Set the log level - concat::fragment { 'log_level': - target => 'consul-template/config.json', - content => inline_template("log_level = \"${::consul_template::log_level}\"\n"), - order => '01' - } + Concat::Fragment <| target == 'consul-template/config.json' |> - # Set wait param if specified - if $::consul_template::consul_wait { - concat::fragment { 'consul_wait': - target => 'consul-template/config.json', - content => inline_template("wait = \"${::consul_template::consul_wait}\"\n\n"), - order => '02', - } - } - - # Set max_stale param if specified - if $::consul_template::consul_max_stale { - concat::fragment { 'consul_max_stale': - target => 'consul-template/config.json', - content => inline_template("max_stale = \"${::consul_template::consul_max_stale}\"\n\n"), - order => '03', - } - } - - if $::consul_template::deduplicate { - concat::fragment { 'dedup-base': - target => 'consul-template/config.json', - content => inline_template("deduplicate {\n enabled = true\n"), - order => '04', - } - - if $::consul_template::deduplicate_prefix { - concat::fragment { 'dedup-prefix': - target => 'consul-template/config.json', - content => inline_template(" prefix = \"${::consul_template::deduplicate_prefix}\"\n"), - order => '05', - } - } - - concat::fragment { 'dedup-close': - target => 'consul-template/config.json', - content => inline_template("}\n"), - order => '06', - } - } - - if $::consul_template::vault_enabled { - concat::fragment { 'vault-base': - target => 'consul-template/config.json', - content => inline_template("vault {\n address = \"${::consul_template::vault_address}\"\n token = \"${::consul_template::vault_token}\"\n"), - order => '07', - } - if $::consul_template::vault_ssl { - concat::fragment { 'vault-ssl1': - target => 'consul-template/config.json', - content => inline_template(" ssl {\n enabled = true\n verify = ${::consul_template::vault_ssl_verify}\n"), - order => '08', - } - concat::fragment { 'vault-ssl2': - target => 'consul-template/config.json', - content => inline_template(" cert = \"${::consul_template::vault_ssl_cert}\"\n ca_cert = \"${::consul_template::vault_ssl_ca_cert}\"\n }\n"), - order => '09', - } - } - concat::fragment { 'vault-baseclose': - target => 'consul-template/config.json', - content => "}\n\n", - order => '10', - } + concat::fragment { 'consul-service-post': + target => $config_file, + content => " ],\n}", + order => '99', } file { [$consul_template::config_dir, "${consul_template::config_dir}/config"]: @@ -97,7 +33,7 @@ class consul_template::config ( mode => '0755', } -> concat { 'consul-template/config.json': - path => "${consul_template::config_dir}/config/config.json", + path => $config_file, owner => $consul_template::user, group => $consul_template::group, mode => $consul_template::config_mode, diff --git a/manifests/init.pp b/manifests/init.pp index 98cea52..eb42981 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -84,27 +84,13 @@ class consul_template ( $extra_options = '', $service_enable = true, $service_ensure = 'running', - $consul_host = 'localhost', - $consul_port = '8500', - $consul_token = '', - $consul_retry = '10s', - $consul_wait = undef, - $consul_max_stale = undef, - $deduplicate = false, - $deduplicate_prefix = undef, + $config_hash = {}, + $config_defaults = {}, $init_style = $consul_template::params::init_style, - $log_level = $consul_template::params::log_level, $logrotate_compress = 'nocompress', $logrotate_files = 4, $logrotate_on = false, $logrotate_period = 'daily', - $vault_enabled = false, - $vault_address = '', - $vault_token = '', - $vault_ssl = true, - $vault_ssl_verify = true, - $vault_ssl_cert = '', - $vault_ssl_ca_cert = '', $data_dir = '', $user = $consul_template::params::user, $group = $consul_template::params::group, @@ -119,6 +105,8 @@ class consul_template ( validate_bool($manage_user) validate_bool($manage_group) validate_hash($watches) + validate_hash($config_hash) + validate_hash($config_defaults) $real_download_url = pick($download_url, "${download_url_base}${version}/${package_name}_${version}_${os}_${arch}.${download_extension}") @@ -126,13 +114,15 @@ class consul_template ( create_resources(consul_template::watch, $watches) } + $config_base = { + consul => 'localhost:8500', + } + $config_hash_real = deep_merge($config_base, $config_defaults, $config_hash) + anchor { '::consul_template::begin': } -> class { '::consul_template::install': } -> class { '::consul_template::config': - consul_host => $consul_host, - consul_port => $consul_port, - consul_token => $consul_token, - consul_retry => $consul_retry, + config_hash => $config_hash_real, purge => $purge_config_dir, } ~> class { '::consul_template::service': } -> diff --git a/manifests/params.pp b/manifests/params.pp index 57a49c7..ee8c3e2 100644 --- a/manifests/params.pp +++ b/manifests/params.pp @@ -6,7 +6,6 @@ class consul_template::params { $install_method = 'url' - $log_level = 'info' $package_name = 'consul-template' $package_ensure = 'latest' $version = '0.11.0' diff --git a/manifests/watch.pp b/manifests/watch.pp index e92b90a..3405421 100644 --- a/manifests/watch.pp +++ b/manifests/watch.pp @@ -41,11 +41,13 @@ define consul_template::watch ( } $config_hash_real = deep_merge($config_hash, $config_source) - $content = consul_sorted_json($config_hash_real, $consul::pretty_config, $consul::pretty_config_indent) - concat::fragment { $frag_name: + $content = consul_sorted_json($config_hash_real, $consul::pretty_config, $consul::pretty_config_indent)[0,-2] + @concat::fragment { $frag_name: target => 'consul-template/config.json', - content => "template ${content}\n\n", - order => '10', - notify => Service['consul-template'] + # NOTE: this will result in all watches having , after them in the JSON + # array. That won't pass strict JSON parsing, but luckily HCL is fine with it, + content => "$content,\n", + order => '50', + notify => Service['consul-template'], } } From 19b7ec17abad0d1cc7ec94343979d058cb723fd9 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 23 Jun 2016 22:15:59 -0700 Subject: [PATCH 3/7] Support config_defaults in consul_template::watch --- manifests/watch.pp | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/manifests/watch.pp b/manifests/watch.pp index 3405421..9240642 100644 --- a/manifests/watch.pp +++ b/manifests/watch.pp @@ -4,24 +4,26 @@ # This is a single instance of a configuration file to watch # for changes in Consul and update the local file define consul_template::watch ( - $config_hash = {}, - $template = undef, - $template_vars = {}, + $config_hash = {}, + $config_defaults = {}, + $template = undef, + $template_vars = {}, ) { include consul_template - if $template == undef and $config_hash['source'] == undef { + $config_hash_real = deep_merge($config_defaults, $config_hash) + if $template == undef and $config_hash_real['source'] == undef { err ('Specify either template parameter or config_hash["source"] for consul_template::watch') } - if $template != undef and $config_hash['source'] != undef { + if $template != undef and $config_hash_real['source'] != undef { err ('Specify either template parameter or config_hash["source"] for consul_template::watch - but not both') } if $template == undef { # source is specified in config_hash $config_source = {} - $frag_name = $config_hash['source'] + $frag_name = $config_hash_real['source'] } else { # source is specified as a template $source = "${consul_template::config_dir}/${name}.ctmpl" @@ -40,8 +42,8 @@ define consul_template::watch ( $frag_name = $source } - $config_hash_real = deep_merge($config_hash, $config_source) - $content = consul_sorted_json($config_hash_real, $consul::pretty_config, $consul::pretty_config_indent)[0,-2] + $config_hash_all = deep_merge($config_hash_real, $config_source) + $content = consul_sorted_json($config_hash_all, $consul::pretty_config, $consul::pretty_config_indent)[0,-2] @concat::fragment { $frag_name: target => 'consul-template/config.json', # NOTE: this will result in all watches having , after them in the JSON From c9ccc192d6d7affac9574f6d3113e3900a62d600 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 24 Jun 2016 07:04:06 -0700 Subject: [PATCH 4/7] Better doc and comments on config_hash --- README.markdown | 21 ++++++++++++++++----- manifests/config.pp | 2 ++ manifests/init.pp | 24 +++--------------------- manifests/watch.pp | 2 +- 4 files changed, 22 insertions(+), 27 deletions(-) diff --git a/README.markdown b/README.markdown index 9958fa9..aa55434 100644 --- a/README.markdown +++ b/README.markdown @@ -4,9 +4,8 @@ This fork is not backwards compatible with its upstream. The configuration mechanism has been modified to be driven by config_hash/config_defaults instead -of individual parameters to provide access to the full set of configuration -options and avoid the need to make changes here as consul-template options -evolve. +of individual parameters to provide access to the full set of options and avoid +the need to make changes here as consul-template options evolve. ##Installation @@ -42,7 +41,7 @@ set to 'package', its installed using the system package manager. - `manage_user` Default: false. Module handles creating the user. - `manage_group` Default: false. Module handles creating the group. - `init_style` Init style to use for consul-template service. -- `config_hash` Default: {}. Consul-template configuration options +- `config_hash` Default: {}. Consul-template configuration options. See https://github.com/hashicorp/consul-template#options - `config_defaults` Default: {}. Consul-template configuration option defaults. @@ -54,7 +53,19 @@ The simplest way to use this module is: include consul_template ``` -Or to specify parameters: +consul-template options can be passed via hiera: + +``` +consul_template::config_defaults: + deduplicate: + enabled: true + log_level: info + retry: 10s + syslog: true + token: +``` + +Or to specify class parameters: ```puppet class { 'consul_template': service_enable => false diff --git a/manifests/config.pp b/manifests/config.pp index c9d0df4..403d1b0 100644 --- a/manifests/config.pp +++ b/manifests/config.pp @@ -16,6 +16,8 @@ class consul_template::config ( order => '1', } + # Realizes concat::fragments from consul_template::watches that make up 1 or + # more template configs. Concat::Fragment <| target == 'consul-template/config.json' |> concat::fragment { 'consul-service-post': diff --git a/manifests/init.pp b/manifests/init.pp index eb42981..97d97b3 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -22,33 +22,15 @@ # [*init_style*] # What style of init system your system uses. # +# [*config_hash*] +# Consul-template configuration options. See https://github.com/hashicorp/consul-template#options +# # [*config_mode*] # Set config file mode # # [*purge_config_dir*] # Purge config files no longer generated by Puppet # -# [*vault_enabled*] -# Do we want to configure Hashicopr Vault? Defaults to false. -# -# [*vault_address*] -# HTTP/HTTPS URL of the vault service -# -# [*vault_token*] -# Auth token to use for Vault -# -# [*vault_ssl*] -# Should we use SSL? Defaults to true -# -# [*vault_ssl_verify*] -# Should we verify the SSL certificate? Defaults to true -# -# [*vault_ssl_cert*] -# What is the path to the cert.pem -# -# [*vault_ssl_ca_cert*] -# What is the path to the ca cert.pem -# # [*data_dir*] # Path to a directory to create to hold some data. Defaults to '' # diff --git a/manifests/watch.pp b/manifests/watch.pp index 9240642..d49ead7 100644 --- a/manifests/watch.pp +++ b/manifests/watch.pp @@ -47,7 +47,7 @@ define consul_template::watch ( @concat::fragment { $frag_name: target => 'consul-template/config.json', # NOTE: this will result in all watches having , after them in the JSON - # array. That won't pass strict JSON parsing, but luckily HCL is fine with it, + # array. That won't pass strict JSON parsing, but luckily HCL is fine with it. content => "$content,\n", order => '50', notify => Service['consul-template'], From ef1fc4e1e5667891381080ee51b0f52f418cb1da Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 24 Jun 2016 09:23:53 -0700 Subject: [PATCH 5/7] PuppetSyntax.future_parser = true --- Rakefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Rakefile b/Rakefile index ea3b9c9..f44d959 100644 --- a/Rakefile +++ b/Rakefile @@ -31,6 +31,7 @@ exclude_paths = [ ] PuppetLint.configuration.ignore_paths = exclude_paths PuppetSyntax.exclude_paths = exclude_paths +PuppetSyntax.future_parser = true desc "Run acceptance tests" RSpec::Core::RakeTask.new(:acceptance) do |t| From 2176a8b1346d0346dd51e6654297194229c70cb0 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 24 Jun 2016 10:02:20 -0700 Subject: [PATCH 6/7] Clean up generated config, use regsubst, well formatted --- manifests/config.pp | 6 +++++- manifests/watch.pp | 5 +++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/manifests/config.pp b/manifests/config.pp index 403d1b0..d9055be 100644 --- a/manifests/config.pp +++ b/manifests/config.pp @@ -7,11 +7,14 @@ class consul_template::config ( $purge = true, ) { # Using our parent module's pretty_config & pretty_config_indent just because - $content = consul_sorted_json($config_hash, $consul::pretty_config, $consul::pretty_config_indent)[0,-1] + $content_full = consul_sorted_json($config_hash, $consul::pretty_config, $consul::pretty_config_indent) + # remove the closing } and it's surrounding newlines + $content = regsubst($content_full, "\n}\n$", '') $config_file = "${consul_template::config_dir}/config/config.json" concat::fragment { 'consul-service-pre': target => $config_file, + # add the opening template array so that we can insert watch fragments content => "${content},\n \"template\": [\n", order => '1', } @@ -22,6 +25,7 @@ class consul_template::config ( concat::fragment { 'consul-service-post': target => $config_file, + # close off the template array and the whole object content => " ],\n}", order => '99', } diff --git a/manifests/watch.pp b/manifests/watch.pp index d49ead7..029891a 100644 --- a/manifests/watch.pp +++ b/manifests/watch.pp @@ -43,12 +43,13 @@ define consul_template::watch ( } $config_hash_all = deep_merge($config_hash_real, $config_source) - $content = consul_sorted_json($config_hash_all, $consul::pretty_config, $consul::pretty_config_indent)[0,-2] + $content_full = consul_sorted_json($config_hash_all, $consul::pretty_config, $consul::pretty_config_indent) + $content = regsubst(regsubst($content_full, "}\n$", '}'), "\n", "\n ", 'G') @concat::fragment { $frag_name: target => 'consul-template/config.json', # NOTE: this will result in all watches having , after them in the JSON # array. That won't pass strict JSON parsing, but luckily HCL is fine with it. - content => "$content,\n", + content => " $content,\n", order => '50', notify => Service['consul-template'], } From e76ca2f5d25b38fbd25d255ed8cb40fac53d2ceb Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 24 Jun 2016 10:43:14 -0700 Subject: [PATCH 7/7] Get spec tests closer to working --- .fixtures.yml | 1 + spec/defines/watch_spec.rb | 14 +++++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.fixtures.yml b/.fixtures.yml index 9d10937..8ad30e2 100644 --- a/.fixtures.yml +++ b/.fixtures.yml @@ -3,5 +3,6 @@ fixtures: stdlib: "git://github.com/puppetlabs/puppetlabs-stdlib.git" staging: "https://github.com/nanliu/puppet-staging.git" concat: "https://github.com/puppetlabs/puppet-concat.git" + consul: "https://github.com/solarkennedy/puppet-consul" symlinks: consul_template: "#{source_dir}" diff --git a/spec/defines/watch_spec.rb b/spec/defines/watch_spec.rb index 618df2c..ea46544 100644 --- a/spec/defines/watch_spec.rb +++ b/spec/defines/watch_spec.rb @@ -7,8 +7,10 @@ describe 'consul_template::watch', :type => :define do describe "consul_template::watch define on OS family #{osfamily}" do let(:title) { 'test_watcher' } let(:params) {{ - :destination => '/var/tmp/consul_template', - :command => '/bin/test', + :config_hash => { + 'destination' => '/var/tmp/consul_template', + 'command' => '/bin/test', + } }} let(:facts) {{ :osfamily => osfamily, @@ -30,10 +32,12 @@ describe 'consul_template::watch', :type => :define do describe "consul_template::watch define on OS family #{osfamily}" do let(:title) { 'test_watcher' } let(:params) {{ - :template => 'consul_template_spec/test_template', + :template => 'consul_template_spec/test_template', :template_vars => { 'foo' => 'bar' }, - :destination => '/var/tmp/consul_template', - :command => '/bin/test', + :config_hash => { + 'destination' => '/var/tmp/consul_template', + 'command' => '/bin/test', + } }} let(:facts) {{ :osfamily => osfamily,