Import s3_file version 2.1.0

jhmartin · Aug 24, 2013 · e657d69 · e657d69
1 parent 5615d26
commit e657d69
Show file tree

Hide file tree

Showing 8 changed files with 223 additions and 0 deletions.
diff --git a/cookbooks/s3_file/.gitignore b/cookbooks/s3_file/.gitignore
@@ -0,0 +1,2 @@
+.DS_Store
+metadata.json
diff --git a/cookbooks/s3_file/MIT-LICENSE.txt b/cookbooks/s3_file/MIT-LICENSE.txt
@@ -0,0 +1,20 @@
+Copyright 2012-2013 Brandon Adams and other contributors
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/cookbooks/s3_file/README.rdoc b/cookbooks/s3_file/README.rdoc
@@ -0,0 +1,44 @@
+= DESCRIPTION:
+An LWRP that can be used to fetch files from S3.
+
+I created this LWRP to solve the chicken-and-egg problem of fetching files from S3 on the first Chef run on a newly provisioned machine. Ruby libraries that are installed on that first run are not available to Chef during the run, so I couldn't use a library like Fog to get what I needed from S3.
+
+This LWRP has no dependencies beyond the Ruby standard library, so it can be used on the first run of Chef.
+
+= REQUIREMENTS:
+An Amazon Web Services account and something in S3 to fetch.
+
+Multi-part S3 uploads do not put the MD5 of the content in the ETag header. If x-amz-meta-digest is provided in User-Defined Metadata on the S3 Object it is processed as if it were a Digest header (RFC 3230).
+
+The MD5 of the local file will be checked against the MD5 from x-amz-meta-digest if it is present.  It not it will check against the ETag.  If there is no match or the local file is absent it will be downloaded.
+
+If credentials are not provided, s3_file will attempt to use the first instance profile associated with the instance. See documentation at http://docs.aws.amazon.com/IAM/latest/UserGuide/instance-profiles.html for more on instance profiles.
+
+= USAGE:
+s3_file acts like other file resources.  The only supported action is :create, which is the default.
+
+Attribute Parameters:
+
+* `aws_access_key_id` - your AWS access key id. (optional)
+* `aws_secret_access_key` - your AWS secret access key. (optional)
+* `token` - token used for temporary IAM credentials. (optional)
+* `bucket` - the bucket to pull from.
+* `remote_path` - the S3 key to pull.
+* `owner` - the owner of the file. (optional)
+* `group` - the group owner of the file. (optional)
+* `mode` - the octal mode of the file. (optional)
+
+Example:
+
+    s3_file "/tmp/somefile" do
+    	remote_path "/my/s3/key"
+    	bucket "my-s3-bucket"
+    	aws_access_key_id "mykeyid"
+    	aws_secret_access_key "mykey"
+    	owner "me"
+    	group "mygroup"
+    	mode "0644"
+    	action :create
+    end
+
+
diff --git a/cookbooks/s3_file/libraries/s3_file.rb b/cookbooks/s3_file/libraries/s3_file.rb
@@ -0,0 +1,63 @@
+require 'rest-client'
+require 'time'
+require 'openssl'
+require 'base64'
+
+module S3FileLib
+  def self.build_headers(date, authorization, token)
+    headers = {
+      :date => date,
+      :authorization => authorization
+    }
+    if token
+      headers['x-amz-security-token'] = token
+    end
+
+    return headers
+  end
+
+  def self.get_md5_from_s3(bucket,path,aws_access_key_id,aws_secret_access_key,token)
+    return get_digests_from_s3(bucket,path,aws_access_key_id,aws_secret_access_key,token)["md5"]
+  end
+
+  def self.get_digests_from_s3(bucket,path,aws_access_key_id,aws_secret_access_key,token)
+    now, auth_string = get_s3_auth("HEAD", bucket,path,aws_access_key_id,aws_secret_access_key, token)
+
+    headers = build_headers(now, auth_string, token)
+    response = RestClient.head('https://%s.s3.amazonaws.com%s' % [bucket,path], headers)
+
+    etag = response.headers[:etag].gsub('"','')
+    digest = response.headers[:x_amz_meta_digest]
+    digests = digest.nil? ? {} : Hash[digest.split(",").map {|a| a.split("=")}]
+
+    return {"md5" => etag}.merge(digests)
+  end
+
+  def self.get_from_s3(bucket,path,aws_access_key_id,aws_secret_access_key,token)   
+    now, auth_string = get_s3_auth("GET", bucket,path,aws_access_key_id,aws_secret_access_key, token)
+
+    headers = build_headers(now, auth_string, token)
+    response = RestClient.get('https://%s.s3.amazonaws.com%s' % [bucket,path], headers)
+
+    return response.body
+  end
+
+  def self.get_s3_auth(method, bucket,path,aws_access_key_id,aws_secret_access_key, token)
+    now = Time.now().utc.strftime('%a, %d %b %Y %H:%M:%S GMT')
+    string_to_sign = "#{method}\n\n\n%s\n" % [now]
+
+    if token
+      string_to_sign += "x-amz-security-token:#{token}\n"
+    end
+
+    string_to_sign += "/%s%s" % [bucket,path]
+
+    digest = digest = OpenSSL::Digest::Digest.new('sha1')
+    signed = OpenSSL::HMAC.digest(digest, aws_secret_access_key, string_to_sign)
+    signed_base64 = Base64.encode64(signed)
+
+    auth_string = 'AWS %s:%s' % [aws_access_key_id,signed_base64]
+
+    [now,auth_string]
+  end
+end
diff --git a/cookbooks/s3_file/metadata.rb b/cookbooks/s3_file/metadata.rb
@@ -0,0 +1,7 @@
+name             "s3_file"
+maintainer       "Brandon Adams"
+maintainer_email "brandon.adams@me.com"
+license          "MIT"
+description      "Installs/Configures s3_file LWRP"
+long_description IO.read(File.join(File.dirname(__FILE__), 'README.rdoc'))
+version          "2.1.0"
diff --git a/cookbooks/s3_file/providers/default.rb b/cookbooks/s3_file/providers/default.rb
@@ -0,0 +1,64 @@
+#include S3File
+
+require 'digest/md5'
+require 'rest-client'
+require 'json'
+
+action :create do
+  new_resource = @new_resource
+  download = true
+
+  # handle key specified without leading slash
+  remote_path = new_resource.remote_path
+  if remote_path.chars.first != '/'
+    remote_path = "/#{remote_path}"
+  end
+
+  # we need credentials to be mutable
+  aws_access_key_id = new_resource.aws_access_key_id
+  aws_secret_access_key = new_resource.aws_secret_access_key
+  token = new_resource.token
+
+  # if credentials not set, try instance profile
+  if new_resource.aws_access_key_id.nil? and new_resource.aws_secret_access_key.nil? and new_resource.token.nil?
+    instance_profile_base_url = 'http://169.254.169.254/latest/meta-data/iam/security-credentials/'
+    begin
+      instance_profiles = RestClient.get(instance_profile_base_url)
+    rescue RestClient::ResourceNotFound, Errno::ETIMEDOUT # we can either 404 on an EC2 instance, or timeout on non-EC2
+      raise ArgumentError.new 'No credentials provided and no instance profile on this machine.'
+    end
+    instance_profile_name = instance_profiles.split.first
+    instance_profile = JSON.load(RestClient.get(instance_profile_base_url + instance_profile_name))
+
+    aws_access_key_id = instance_profile['AccessKeyId']
+    aws_secret_access_key = instance_profile['SecretAccessKey']
+    token = instance_profile['Token']
+  end
+
+  if ::File.exists? new_resource.path
+    s3_md5 = S3FileLib::get_md5_from_s3(new_resource.bucket, remote_path, aws_access_key_id, aws_secret_access_key, token)
+
+    current_md5 = Digest::MD5.hexdigest(::File.read(new_resource.path))
+
+    Chef::Log.debug "md5 of S3 object is #{s3_md5}"
+    Chef::Log.debug "md5 of local object is #{current_md5}"
+
+    if current_md5 == s3_md5 then
+      Chef::Log.debug 'Skipping download, md5sum of local file matches file in S3.'
+      download = false
+    end
+  end
+
+  if download
+    body = S3FileLib::get_from_s3(new_resource.bucket, remote_path, aws_access_key_id, aws_secret_access_key, token).body
+
+    file new_resource.path do
+      owner new_resource.owner if new_resource.owner
+      group new_resource.group if new_resource.group
+      mode new_resource.mode if new_resource.mode
+      action :create
+      content body
+    end
+    @new_resource.updated_by_last_action(true)
+  end
+end
diff --git a/cookbooks/s3_file/recipes/default.rb b/cookbooks/s3_file/recipes/default.rb
@@ -0,0 +1,8 @@
+#
+# Cookbook Name:: s3_file
+# Recipe:: default
+#
+# Copyright 2011, YOUR_COMPANY_NAME
+#
+# All rights reserved - Do Not Redistribute
+#
diff --git a/cookbooks/s3_file/resources/default.rb b/cookbooks/s3_file/resources/default.rb
@@ -0,0 +1,15 @@
+actions :create
+attribute :path, :kind_of => String, :name_attribute => true
+attribute :remote_path, :kind_of => String
+attribute :bucket, :kind_of => String
+attribute :aws_access_key_id, :kind_of => String, :default => nil
+attribute :aws_secret_access_key, :kind_of => String, :default => nil
+attribute :token, :kind_of => String, :default => nil
+attribute :owner, :kind_of => [String, NilClass], :default => nil
+attribute :group, :kind_of => [String, NilClass], :default => nil
+attribute :mode, :kind_of => [String, NilClass], :default => nil
+
+def initialize(*args)
+  super
+  @action = :create
+end