Feature: allow muting a host during an MSI install (#778)

MSI installs can take a long time, during which the Agent is not running. On upgrades, this could trigger no-data monitors. This PR adds a new option to mute a host while an MSI install takes place, thus preventing this issue. The host will unmute after either the install completes, the chef run fails, or after the mute expires (in 60 minutes). It uses the Datadog API, which requires an application_key.
DataDog · Feb 15, 2021 · 97ce7e5 · 97ce7e5
1 parent c12ea59
commit 97ce7e5
Show file tree

Hide file tree

Showing 4 changed files with 119 additions and 0 deletions.
diff --git a/Rakefile b/Rakefile
@@ -29,6 +29,7 @@ namespace :style do
       fail_tags: ['correctness'],
       tags: [
         '~FC121', # Disables: Cookbook depends on cookbook made obsolete by Chef 14 (chef_handler)
+        '~FC014', # Disables: Consider extracting long ruby_block to library
       ]
     }
   end

diff --git a/attributes/default.rb b/attributes/default.rb
@@ -189,6 +189,11 @@
 # Default should suffice, but provides a knob in case instances with limited resources timeout.
 default['datadog']['windows_msi_timeout'] = 1200
 
+# Mute hosts during an MSI installation
+# To prevent no-data alerts when MSI installs take long.
+# Requires setting 'application_key' to a valid app key for the Datadog REST API.
+default['datadog']['windows_mute_hosts_during_install'] = false
+
 # Agent installer checksum
 # Expected checksum to validate correct agent installer is downloaded (Windows only)
 default['datadog']['windows_agent_checksum'] = nil

diff --git a/recipes/_install-windows.rb b/recipes/_install-windows.rb
@@ -36,6 +36,11 @@ def clean_password(context)
       resource = context.resource_collection.lookup('windows_env[DDAGENTUSER_PASSWORD]')
       resource.run_action(:delete)
     end
+
+    def unmute_host(context)
+      resource = context.resource_collection.lookup('ruby_block[Unmute host after installing]')
+      resource.run_action(:run)
+    end
   end
 end
 
@@ -157,6 +162,60 @@ def clean_password(context)
   notifies :remove, 'package[Datadog Agent removal]', :immediately
 end
 
+if node['datadog']['windows_mute_hosts_during_install']
+  unless Chef::Datadog.application_key(node)
+    Chef::Log.error('windows_mute_hosts_during_install requires an application_key but it is not set.')
+  end
+  ruby_block 'Mute host while installing' do
+    block do
+      require 'net/http'
+      require 'uri'
+      require 'json'
+      hostname = node['datadog']['hostname']
+      uri = URI.parse("https://api.datadoghq.com/api/v1/host/#{hostname}/mute")
+      request = Net::HTTP::Post.new(uri)
+      request.content_type = 'application/json'
+      request['Dd-Api-Key'] = Chef::Datadog.api_key(node)
+      request['Dd-Application-Key'] = Chef::Datadog.application_key(node)
+      request.body = JSON.dump({
+        'message': 'Muted during install by datadog-chef cookbook',
+        'end': Time.now.getutc.to_i + (60 * 60), # Set to automatically unmute in 60 minutes
+      })
+      response = Net::HTTP.start(uri.hostname, uri.port, { use_ssl: true }) do |http|
+        http.request(request)
+      end
+      if response.code != '200'
+        Chef::Log.warn("Mute request failed with code #{response.code}: #{response.body}")
+      end
+    end
+    action :nothing
+  end
+  ruby_block 'Unmute host after installing' do
+    block do
+      require 'net/http'
+      require 'uri'
+      require 'json'
+      hostname = node['datadog']['hostname']
+      uri = URI.parse("https://api.datadoghq.com/api/v1/host/#{hostname}/unmute")
+      request = Net::HTTP::Post.new(uri)
+      request['Dd-Api-Key'] = Chef::Datadog.api_key(node)
+      request['Dd-Application-Key'] = Chef::Datadog.application_key(node)
+      response = Net::HTTP.start(uri.hostname, uri.port, { use_ssl: true }) do |http|
+        http.request(request)
+      end
+      if response.code != '200'
+        Chef::Log.warn("Unmute request failed with code #{response.code}: #{response.body}")
+      end
+    end
+    action :nothing
+  end
+  Chef.event_handler do
+    on :run_failed do
+      Windows::Helper.new.unmute_host(Chef.run_context)
+    end
+  end
+end
+
 # Install the package
 windows_package 'Datadog Agent' do # ~FC009
   source temp_file
@@ -171,6 +230,10 @@ def clean_password(context)
   else
     success_codes [0, 3010]
   end
+  if node['datadog']['windows_mute_hosts_during_install']
+    notifies :run, 'ruby_block[Mute host while installing]', :before
+    notifies :run, 'ruby_block[Unmute host after installing]', :immediately
+  end
   not_if do
     require 'digest'
 

diff --git a/spec/dd-agent_spec.rb b/spec/dd-agent_spec.rb
@@ -1326,6 +1326,56 @@ def set_env_var(name, value)
     end
   end
 
+  context 'windows_mute_hosts_during_install' do
+    cached(:chef_run) do
+      ChefSpec::SoloRunner.new(
+        :platform => 'windows',
+        :version => '2012R2'
+      ) do |node|
+        node.normal['datadog'] = {
+          'api_key' => 'somethingnotnil',
+          'windows_mute_hosts_during_install' => true,
+          'agent_major_version' => 7,
+        }
+      end.converge described_recipe
+    end
+
+    it 'mutes and unmutes host' do
+      expect(chef_run).to nothing_ruby_block('Mute host while installing')
+      expect(chef_run).to nothing_ruby_block('Unmute host after installing')
+      expect(chef_run).to install_windows_package('Datadog Agent').with(installer_type: :msi)
+
+      package = chef_run.windows_package('Datadog Agent')
+      expect(package).to notify('ruby_block[Mute host while installing]').to(:run).before
+      expect(package).to notify('ruby_block[Unmute host after installing]').to(:run).immediately
+    end
+  end
+
+  context 'not windows_mute_hosts_during_install' do
+    cached(:chef_run) do
+      ChefSpec::SoloRunner.new(
+        :platform => 'windows',
+        :version => '2012R2'
+      ) do |node|
+        node.normal['datadog'] = {
+          'api_key' => 'somethingnotnil',
+          'windows_mute_hosts_during_install' => false,
+          'agent_major_version' => 7,
+        }
+      end.converge described_recipe
+    end
+
+    it 'mutes and unmutes host' do
+      expect(chef_run).to_not nothing_ruby_block('Mute host while installing')
+      expect(chef_run).to_not nothing_ruby_block('Unmute host after installing')
+      expect(chef_run).to install_windows_package('Datadog Agent').with(installer_type: :msi)
+
+      package = chef_run.windows_package('Datadog Agent')
+      expect(package).to_not notify('ruby_block[Mute host while installing]').to(:run).before
+      expect(package).to_not notify('ruby_block[Unmute host after installing]').to(:run).immediately
+    end
+  end
+
   context 'add prefix and suffix to version number in debian' do
     let(:chef_run) do
       ChefSpec::SoloRunner.new(
-Original file line number
+Diff line change
@@ Expand Up / @@ -29,6 +29,7 @@ namespace :style do @@
           fail_tags: ['correctness'],
           tags: [
             '~FC121', # Disables: Cookbook depends on cookbook made obsolete by Chef 14 (chef_handler)
+            '~FC014', # Disables: Consider extracting long ruby_block to library
           ]
         }
       end
@@ Expand Down @@