[run_state] Improve CI, documentation and backwards compatibility

CI and documentation: * make rubocop pass, mostly by disabling some cops on a few blocks/methods where I couldn't find good alternatives. * move the extra_config logic to the `datadog.conf.erb` template. * add spec tests * document pulling api/app keys from node `run_state` in README and default attributes file BC improvemnts: * rename `attr` to `attribute` in `recipe_helpers.rb` * keep raising an exception when no `api_key` is present (at converge time instead of compile time)
DataDog · olivielpeau · Jan 24, 2017 · Aug 12, 2016 · Aug 12, 2016 · Jan 20, 2017
commit d20d9b989186b99a72d6320ee3f2a22aaa9e306e
diff --git a/.rubocop.yml b/.rubocop.yml
@@ -19,6 +19,11 @@ Metrics/BlockLength:
     - 'spec/**/*.rb'
     - 'test/**/*.rb'
 
+
+# Slightly longer methods are fine
+Metrics/MethodLength:
+  Max: 15
+
 # TODO: add encoding headers
 Style/Encoding:
   Enabled: false

diff --git a/README.md b/README.md
@@ -16,6 +16,9 @@ Datadog Cookbook
 
 Chef recipes to deploy Datadog's components and configuration automatically.
 
+**NB: This README may refer to features that are not released yet. Please check the README of the
+git tag/the gem version you're using for your version's documentation**
+
 Requirements
 ============
 - chef >= 10.14
@@ -86,10 +89,16 @@ Usage
 
 1. Add this cookbook to your Chef Server, either by installing with knife or by adding it to your Berksfile:
   ```
-  cookbook 'datadog', '~> 2.1.0'
+  cookbook 'datadog', '~> 2.7.0'
   ```
-2. Add your API Key as a node attribute via an `environment` or `role` or by declaring it in another cookbook at a higher precedence level.
-3. Create an 'application key' for `chef_handler` [here](https://app.datadoghq.com/account/settings#api), and add it as a node attribute, as in Step #2.
+2. Add your API Key either:
+  * as a node attribute via an `environment` or `role`, or
+  * as a node attribute by declaring it in another cookbook at a higher precedence level, or
+  * in the node `run_state` by setting `node.run_state['datadog']['api_key']` in another cookbook preceding `datadog`'s recipes in the run_list. This approach has the benefit of not storing the credential in clear text on the Chef Server.
+3. Create an 'application key' for `chef_handler` [here](https://app.datadoghq.com/account/settings#api), and add it as a node attribute or in the run state, as in Step #2.
+
+   NB: if you're using the run state to store the api and app keys you need to set them at compile time before `datadog::dd-handler` in the run list.
+
 4. Associate the recipes with the desired `roles`, i.e. "role:chef-client" should contain "datadog::dd-handler" and a "role:base" should start the agent with "datadog::dd-agent".  Here's an example role with both recipes:
   ```
   name 'example'

diff --git a/attributes/default.rb b/attributes/default.rb
@@ -17,13 +17,15 @@
 # limitations under the License.
 #
 
-# Place your API Key here, or set it on the role/environment/node
+# Place your API Key here, or set it on the role/environment/node, or set it on your
+# node `run_state` under the key `['datadog']['api_key']`.
 # The Datadog api key to associate your agent's data with your organization.
 # Can be found here:
 # https://app.datadoghq.com/account/settings
 default['datadog']['api_key'] = nil
 
-# Create an application key on the Account Settings page
+# Create an application key on the Account Settings page.
+# Set it as an attribute, or on your node `run_state` under the key `['datadog']['application_key']`
 default['datadog']['application_key'] = nil
 
 # Use this attribute to send data to additional accounts

diff --git a/libraries/recipe_helpers.rb b/libraries/recipe_helpers.rb
@@ -12,11 +12,11 @@ def application_key(node)
 
       private
 
-      def run_state_or_attribute(node, attr)
-        if node.run_state.key?('datadog') && node.run_state['datadog'].key?(attr)
-          node.run_state['datadog'][attr]
+      def run_state_or_attribute(node, attribute)
+        if node.run_state.key?('datadog') && node.run_state['datadog'].key?(attribute)
+          node.run_state['datadog'][attribute]
         else
-          node['datadog'][attr]
+          node['datadog'][attribute]
         end
       end
     end

diff --git a/recipes/dd-agent.rb b/recipes/dd-agent.rb
@@ -17,6 +17,14 @@
 # limitations under the License.
 #
 
+# Fail here at converge time if no api_key is set
+ruby_block 'datadog-api-key-unset' do
+  block do
+    raise "Set ['datadog']['api_key'] as an attribute or on the node's run_state to configure this node's Datadog Agent."
+  end
+  only_if { Chef::Datadog.api_key(node).nil? }
+end
+
 is_windows = node['platform_family'] == 'windows'
 
 # Install the agent
@@ -52,9 +60,8 @@
 # the node's run_list and set the relevant attributes
 #
 
-
-template agent_config_file do
-  def template_vars
+template agent_config_file do # rubocop:disable Metrics/BlockLength
+  def template_vars # rubocop:disable Metrics/AbcSize
     api_keys = [Chef::Datadog.api_key(node)]
     dd_urls = [node['datadog']['url']]
     node['datadog']['extra_endpoints'].each do |_, endpoint|
@@ -66,15 +73,9 @@ def template_vars
                    node['datadog']['url']
                  end
     end
-    extra_config = {}
-    node['datadog']['extra_config'].each do |option, value|
-      next if value.nil?
-      extra_config[option] = value
-    end
     {
       :api_keys => api_keys,
-      :dd_urls => dd_urls,
-      :extra_config => extra_config
+      :dd_urls => dd_urls
     }
   end
   if is_windows

diff --git a/recipes/dd-handler.rb b/recipes/dd-handler.rb
@@ -47,16 +47,19 @@
 end
 
 # Create the handler to run at the end of the Chef execution
-chef_handler 'Chef::Handler::Datadog' do
-  def handler_config # rubocop:disable Metrics/AbcSize
+chef_handler 'Chef::Handler::Datadog' do # rubocop:disable Metrics/BlockLength
+  def extra_endpoints
     extra_endpoints = []
     node['datadog']['extra_endpoints'].each do |_, endpoint|
       next unless endpoint['enabled']
       endpoint = Mash.new(endpoint)
       endpoint.delete('enabled')
       extra_endpoints << endpoint
     end
+    extra_endpoints
+  end
 
+  def handler_config # rubocop:disable Metrics/AbcSize
     config = {
       :api_key => Chef::Datadog.api_key(node),
       :application_key => Chef::Datadog.application_key(node),

diff --git a/spec/dd-agent_spec.rb b/spec/dd-agent_spec.rb
@@ -76,6 +76,7 @@ def set_env_var(name, value)
   include EnvVar
 
   context 'no version set' do
+    # This recipe needs to have an api_key, otherwise `raise` is called.
     # It also depends on the version of Python present on the platform:
     #   2.6 and up => datadog-agent is installed
     #   below 2.6 => datadog-agent-base is installed
@@ -372,7 +373,9 @@ def set_env_var(name, value)
 
       it_behaves_like 'debianoids repo'
     end
+  end
 
+  context 'datadog.conf configuration' do
     context 'allows a string for tags' do
       cached(:chef_run) do
         ChefSpec::SoloRunner.new(
@@ -497,6 +500,67 @@ def set_env_var(name, value)
           .with_content(%r{^dd_url: https://app.example.com,https://app.example.com$})
       end
     end
+
+    context 'with no api_key set' do
+      cached(:chef_run) do
+        ChefSpec::SoloRunner.new(
+          platform: 'ubuntu',
+          version: '12.04'
+        ) do |node|
+          node.set['languages'] = { 'python' => { 'version' => '2.6.2' } }
+        end.converge described_recipe
+      end
+
+      it 'raises an error' do
+        expect(chef_run).to run_ruby_block('datadog-api-key-unset')
+        expect { chef_run.ruby_block('datadog-api-key-unset').old_run_action(:run) }.to raise_error(RuntimeError)
+      end
+    end
+
+    context 'with api_key set as node attribute and on node run_state' do
+      cached(:chef_run) do
+        ChefSpec::SoloRunner.new(
+          platform: 'ubuntu',
+          version: '12.04'
+        ) do |node|
+          node.set['datadog'] = {
+            'api_key' => 'as_node_attribute',
+            'url' => 'https://app.example.com'
+          }
+          node.set['languages'] = { 'python' => { 'version' => '2.6.2' } }
+          node.run_state['datadog'] = { 'api_key' => 'on_run_state' }
+        end.converge described_recipe
+      end
+
+      it_behaves_like 'common linux resources'
+
+      it 'uses the api_key from the run_state' do
+        expect(chef_run).to render_file('/etc/dd-agent/datadog.conf')
+          .with_content(/^api_key: on_run_state$/)
+
+        expect(chef_run).not_to render_file('/etc/dd-agent/datadog.conf')
+          .with_content(/api_key: as_node_attribute/)
+      end
+    end
+
+    context 'with api_key set on node run_state only' do
+      cached(:chef_run) do
+        ChefSpec::SoloRunner.new(
+          platform: 'ubuntu',
+          version: '12.04'
+        ) do |node|
+          node.set['languages'] = { 'python' => { 'version' => '2.6.2' } }
+          node.run_state['datadog'] = { 'api_key' => 'on_run_state' }
+        end.converge described_recipe
+      end
+
+      it_behaves_like 'common linux resources'
+
+      it 'uses the api_key from the run_state' do
+        expect(chef_run).to render_file('/etc/dd-agent/datadog.conf')
+          .with_content(/^api_key: on_run_state$/)
+      end
+    end
   end
 
   context 'does accept extra config options' do

diff --git a/spec/dd-handler_spec.rb b/spec/dd-handler_spec.rb
@@ -68,6 +68,48 @@
     end
   end
 
+  context 'with api and app keys set as node attributes and on node run_state' do
+    cached(:chef_run) do
+      ChefSpec::SoloRunner.new(
+        platform: 'ubuntu',
+        version: '14.04'
+      ) do |node|
+        node.set['datadog']['api_key'] = 'api_overriden_by_run_state'
+        node.set['datadog']['application_key'] = 'app_overriden_by_run_state'
+        node.set['datadog']['chef_handler_enable'] = true
+        node.set['datadog']['use_ec2_instance_id'] = true
+        node.run_state['datadog'] = {
+          'api_key' => 'somethingnotnil',
+          'application_key' => 'somethingnotnil2'
+        }
+      end.converge described_recipe
+    end
+
+    it_behaves_like 'a chef-handler-datadog installer'
+
+    it_behaves_like 'a chef-handler-datadog runner'
+  end
+
+  context 'with keys set on node run_state only' do
+    cached(:chef_run) do
+      ChefSpec::SoloRunner.new(
+        platform: 'ubuntu',
+        version: '14.04'
+      ) do |node|
+        node.set['datadog']['chef_handler_enable'] = true
+        node.set['datadog']['use_ec2_instance_id'] = true
+        node.run_state['datadog'] = {
+          'api_key' => 'somethingnotnil',
+          'application_key' => 'somethingnotnil2'
+        }
+      end.converge described_recipe
+    end
+
+    it_behaves_like 'a chef-handler-datadog installer'
+
+    it_behaves_like 'a chef-handler-datadog runner'
+  end
+
   context 'multiple endpoints' do
     cached(:chef_run) do
       ChefSpec::SoloRunner.new(
@@ -93,6 +135,7 @@
 
     it_behaves_like 'a chef-handler-datadog runner', extra_endpoints, nil
   end
+
   context 'multiple endpoints disabled' do
     cached(:chef_run) do
       ChefSpec::SoloRunner.new(

diff --git a/spec/shared_examples.rb b/spec/shared_examples.rb
@@ -8,8 +8,15 @@
   end
 end
 
+shared_examples_for 'datadog conf' do
+  it 'does not complain about a missing api key' do
+    expect(chef_run).not_to run_ruby_block('datadog-api-key-unset')
+  end
+end
+
 shared_examples_for 'common linux resources' do
   it_behaves_like 'datadog-agent service'
+  it_behaves_like 'datadog conf'
 
   it 'includes the repository recipe' do
     expect(chef_run).to include_recipe('datadog::repository')
@@ -50,6 +57,7 @@
 
 shared_examples_for 'common windows resources' do
   it_behaves_like 'datadog-agent service'
+  it_behaves_like 'datadog conf'
 
   it 'ensures the Datadog config directory exists' do
     expect(chef_run).to create_directory 'C:\ProgramData/Datadog'

diff --git a/templates/default/datadog.conf.erb b/templates/default/datadog.conf.erb
@@ -141,11 +141,13 @@ if node['datadog']['legacy_integrations']
 end
 -%>
 
-<% if ! @extra_config.empty? -%>
+<% if ! node['datadog']['extra_config'].empty? -%>
 # ========================================================================== #
 # Other config options
 # ========================================================================== #
-  <% @extra_config.each do |k, v| -%>
+  <% node['datadog']['extra_config'].each do |k, v| -%>
+    <% if ! v.nil? -%>
 <%= k %>: <%= v %>
+    <% end -%>
   <% end -%>
 <% end -%>