Skip to content

Commit

Permalink
Introduce custom metadata
Browse files Browse the repository at this point in the history
  • Loading branch information
@joshuamsager
joshuamsager committed Nov 23, 2021
1 parent 1a06f5d commit e106a4a
Show file tree
Hide file tree
Showing 12 changed files with 140 additions and 40 deletions.
4 changes: 4 additions & 0 deletions activestorage/CHANGELOG.md
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
* Setting custom metadata on blobs are now persisted to remote storage.

*joshuamsager*

* Support direct uploads to multiple services.

*Dmitry Tsepelev*
Expand Down
20 changes: 14 additions & 6 deletions activestorage/app/models/active_storage/blob.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ class ActiveStorage::Blob < ActiveStorage::Record
self.service_name ||= self.class.service&.name
end

after_update_commit :update_service_metadata, if: :content_type_previously_changed?
after_update_commit :update_service_metadata, if: -> { content_type_previously_changed? || metadata_previously_changed? }

before_destroy(prepend: true) do
raise ActiveRecord::InvalidForeignKey if attachments.exists?
Expand Down Expand Up @@ -168,6 +168,14 @@ def filename
ActiveStorage::Filename.new(self[:filename])
end

def custom_metadata
self[:metadata][:custom] || {}
end

def custom_metadata=(metadata)
self[:metadata] = self[:metadata].merge(custom: metadata)
end

# Returns true if the content_type of this blob is in the image range, like image/png.
def image?
content_type.start_with?("image")
Expand Down Expand Up @@ -200,12 +208,12 @@ def url(expires_in: ActiveStorage.service_urls_expire_in, disposition: :inline,
# Returns a URL that can be used to directly upload a file for this blob on the service. This URL is intended to be
# short-lived for security and only generated on-demand by the client-side JavaScript responsible for doing the uploading.
def service_url_for_direct_upload(expires_in: ActiveStorage.service_urls_expire_in)
service.url_for_direct_upload key, expires_in: expires_in, content_type: content_type, content_length: byte_size, checksum: checksum
service.url_for_direct_upload key, expires_in: expires_in, content_type: content_type, content_length: byte_size, checksum: checksum, custom_metadata: custom_metadata
end

# Returns a Hash of headers for +service_url_for_direct_upload+ requests.
def service_headers_for_direct_upload
service.headers_for_direct_upload key, filename: filename, content_type: content_type, content_length: byte_size, checksum: checksum
service.headers_for_direct_upload key, filename: filename, content_type: content_type, content_length: byte_size, checksum: checksum, custom_metadata: custom_metadata
end

def content_type_for_serving # :nodoc:
Expand Down Expand Up @@ -362,11 +370,11 @@ def web_image?

def service_metadata
if forcibly_serve_as_binary?
{ content_type: ActiveStorage.binary_content_type, disposition: :attachment, filename: filename }
{ content_type: ActiveStorage.binary_content_type, disposition: :attachment, filename: filename, custom_metadata: custom_metadata }
elsif !allowed_inline?
{ content_type: content_type, disposition: :attachment, filename: filename }
{ content_type: content_type, disposition: :attachment, filename: filename, custom_metadatata: custom_metadata }
else
{ content_type: content_type }
{ content_type: content_type, custom_metadata: custom_metadata }
end
end

Expand Down
7 changes: 5 additions & 2 deletions activestorage/lib/active_storage/service.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,12 +128,12 @@ def url(key, **options)
# The URL will be valid for the amount of seconds specified in +expires_in+.
# You must also provide the +content_type+, +content_length+, and +checksum+ of the file
# that will be uploaded. All these attributes will be validated by the service upon upload.
def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:)
def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:, custom_metadata: {})
raise NotImplementedError
end

# Returns a Hash of headers for +url_for_direct_upload+ requests.
def headers_for_direct_upload(key, filename:, content_type:, content_length:, checksum:)
def headers_for_direct_upload(key, filename:, content_type:, content_length:, checksum:, custom_metadata: {})
{}
end

Expand All @@ -150,6 +150,9 @@ def public_url(key, **)
raise NotImplementedError
end

def custom_metadata_headers(metadata)
raise NotImplementedError
end

def instrument(operation, payload = {}, &block)
ActiveSupport::Notifications.instrument(
Expand Down
14 changes: 9 additions & 5 deletions activestorage/lib/active_storage/service/azure_storage_service.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,12 @@ def initialize(storage_account_name:, storage_access_key:, container:, public: f
@public = public
end

def upload(key, io, checksum: nil, filename: nil, content_type: nil, disposition: nil, **)
def upload(key, io, checksum: nil, filename: nil, content_type: nil, disposition: nil, custom_metadata: {}, **)
instrument :upload, key: key, checksum: checksum do
handle_errors do
content_disposition = content_disposition_with(filename: filename, type: disposition) if disposition && filename

client.create_block_blob(container, key, IO.try_convert(io) || io, content_md5: checksum, content_type: content_type, content_disposition: content_disposition)
client.create_block_blob(container, key, IO.try_convert(io) || io, content_md5: checksum, content_type: content_type, content_disposition: content_disposition, metadata: custom_metadata)
end
end
end
Expand Down Expand Up @@ -86,7 +86,7 @@ def exist?(key)
end
end

def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:)
def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:, custom_metadata: {})
instrument :url, key: key do |payload|
generated_url = signer.signed_uri(
uri_for(key), false,
Expand All @@ -101,10 +101,10 @@ def url_for_direct_upload(key, expires_in:, content_type:, content_length:, chec
end
end

def headers_for_direct_upload(key, content_type:, checksum:, filename: nil, disposition: nil, **)
def headers_for_direct_upload(key, content_type:, checksum:, filename: nil, disposition: nil, custom_metadata:, **)
content_disposition = content_disposition_with(type: disposition, filename: filename) if filename

{ "Content-Type" => content_type, "Content-MD5" => checksum, "x-ms-blob-content-disposition" => content_disposition, "x-ms-blob-type" => "BlockBlob" }
{ "Content-Type" => content_type, "Content-MD5" => checksum, "x-ms-blob-content-disposition" => content_disposition, "x-ms-blob-type" => "BlockBlob", **custom_metadata_headers(custom_metadata) }
end

private
Expand Down Expand Up @@ -166,5 +166,9 @@ def handle_errors
raise
end
end

def custom_metadata_headers(metadata)
metadata.transform_keys { |key| "x-ms-meta-#{key}" }
end
end
end
2 changes: 1 addition & 1 deletion activestorage/lib/active_storage/service/disk_service.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ def exist?(key)
end
end

def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:)
def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:, custom_metadata: {})
instrument :url, key: key do |payload|
verified_token_with_expiration = ActiveStorage.verifier.generate(
{
Expand Down
20 changes: 13 additions & 7 deletions activestorage/lib/active_storage/service/gcs_service.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,14 @@ def initialize(public: false, **config)
@public = public
end

def upload(key, io, checksum: nil, content_type: nil, disposition: nil, filename: nil)
def upload(key, io, checksum: nil, content_type: nil, disposition: nil, filename: nil, custom_metadata: {})
instrument :upload, key: key, checksum: checksum do
# GCS's signed URLs don't include params such as response-content-type response-content_disposition
# in the signature, which means an attacker can modify them and bypass our effort to force these to
# binary and attachment when the file's content type requires it. The only way to force them is to
# store them as object's metadata.
content_disposition = content_disposition_with(type: disposition, filename: filename) if disposition && filename
bucket.create_file(io, key, md5: checksum, cache_control: @config[:cache_control], content_type: content_type, content_disposition: content_disposition)
bucket.create_file(io, key, md5: checksum, cache_control: @config[:cache_control], content_type: content_type, content_disposition: content_disposition, metadata: custom_metadata)
rescue Google::Cloud::InvalidArgumentError
raise ActiveStorage::IntegrityError
end
Expand All @@ -43,11 +43,12 @@ def download(key, &block)
end
end

def update_metadata(key, content_type:, disposition: nil, filename: nil)
def update_metadata(key, content_type:, disposition: nil, filename: nil, custom_metadata: {})
instrument :update_metadata, key: key, content_type: content_type, disposition: disposition do
file_for(key).update do |file|
file.content_type = content_type
file.content_disposition = content_disposition_with(type: disposition, filename: filename) if disposition && filename
file.metadata = custom_metadata
end
end
end
Expand Down Expand Up @@ -86,7 +87,7 @@ def exist?(key)
end
end

def url_for_direct_upload(key, expires_in:, checksum:, **)
def url_for_direct_upload(key, expires_in:, checksum:, custom_metadata: {}, **)
instrument :url, key: key do |payload|
headers = {}
version = :v2
Expand All @@ -99,6 +100,8 @@ def url_for_direct_upload(key, expires_in:, checksum:, **)
version = :v4
end

headers.merge!(custom_metadata_headers(custom_metadata))

args = {
content_md5: checksum,
expires: expires_in,
Expand All @@ -120,11 +123,10 @@ def url_for_direct_upload(key, expires_in:, checksum:, **)
end
end

def headers_for_direct_upload(key, checksum:, filename: nil, disposition: nil, **)
def headers_for_direct_upload(key, checksum:, filename: nil, disposition: nil, custom_metadata: {}, **)
content_disposition = content_disposition_with(type: disposition, filename: filename) if filename

headers = { "Content-MD5" => checksum, "Content-Disposition" => content_disposition }

headers = { "Content-MD5" => checksum, "Content-Disposition" => content_disposition, **custom_metadata_headers(custom_metadata) }
if @config[:cache_control].present?
headers["Cache-Control"] = @config[:cache_control]
end
Expand Down Expand Up @@ -223,5 +225,9 @@ def signer
response.signed_blob
end
end

def custom_metadata_headers(metadata)
metadata.transform_keys { |key| "x-goog-meta-#{key}" }
end
end
end
26 changes: 15 additions & 11 deletions activestorage/lib/active_storage/service/s3_service.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,14 @@ def initialize(bucket:, upload: {}, public: false, **options)
@upload_options[:acl] = "public-read" if public?
end

def upload(key, io, checksum: nil, filename: nil, content_type: nil, disposition: nil, **)
def upload(key, io, checksum: nil, filename: nil, content_type: nil, disposition: nil, custom_metadata: {}, **)
instrument :upload, key: key, checksum: checksum do
content_disposition = content_disposition_with(filename: filename, type: disposition) if disposition && filename

if io.size < multipart_upload_threshold
upload_with_single_part key, io, checksum: checksum, content_type: content_type, content_disposition: content_disposition
upload_with_single_part key, io, checksum: checksum, content_type: content_type, content_disposition: content_disposition, custom_metadata: custom_metadata
else
upload_with_multipart key, io, content_type: content_type, content_disposition: content_disposition
upload_with_multipart key, io, content_type: content_type, content_disposition: content_disposition, custom_metadata: custom_metadata
end
end
end
Expand Down Expand Up @@ -77,22 +77,22 @@ def exist?(key)
end
end

def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:)
def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:, custom_metadata: {})
instrument :url, key: key do |payload|
generated_url = object_for(key).presigned_url :put, expires_in: expires_in.to_i,
content_type: content_type, content_length: content_length, content_md5: checksum,
whitelist_headers: ["content-length"], **upload_options
metadata: custom_metadata, whitelist_headers: ["content-length"], **upload_options

payload[:url] = generated_url

generated_url
end
end

def headers_for_direct_upload(key, content_type:, checksum:, filename: nil, disposition: nil, **)
def headers_for_direct_upload(key, content_type:, checksum:, filename: nil, disposition: nil, custom_metadata: {}, **)
content_disposition = content_disposition_with(type: disposition, filename: filename) if filename

{ "Content-Type" => content_type, "Content-MD5" => checksum, "Content-Disposition" => content_disposition }
{ "Content-Type" => content_type, "Content-MD5" => checksum, "Content-Disposition" => content_disposition, **custom_metadata_headers(custom_metadata) }
end

private
Expand All @@ -110,16 +110,16 @@ def public_url(key, **client_opts)
MAXIMUM_UPLOAD_PARTS_COUNT = 10000
MINIMUM_UPLOAD_PART_SIZE = 5.megabytes

def upload_with_single_part(key, io, checksum: nil, content_type: nil, content_disposition: nil)
object_for(key).put(body: io, content_md5: checksum, content_type: content_type, content_disposition: content_disposition, **upload_options)
def upload_with_single_part(key, io, checksum: nil, content_type: nil, content_disposition: nil, custom_metadata: {})
object_for(key).put(body: io, content_md5: checksum, content_type: content_type, content_disposition: content_disposition, metadata: custom_metadata, **upload_options)
rescue Aws::S3::Errors::BadDigest
raise ActiveStorage::IntegrityError
end

def upload_with_multipart(key, io, content_type: nil, content_disposition: nil)
def upload_with_multipart(key, io, content_type: nil, content_disposition: nil, custom_metadata: {})
part_size = [ io.size.fdiv(MAXIMUM_UPLOAD_PARTS_COUNT).ceil, MINIMUM_UPLOAD_PART_SIZE ].max

object_for(key).upload_stream(content_type: content_type, content_disposition: content_disposition, part_size: part_size, **upload_options) do |out|
object_for(key).upload_stream(content_type: content_type, content_disposition: content_disposition, part_size: part_size, metadata: custom_metadata, **upload_options) do |out|
IO.copy_stream(io, out)
end
end
Expand All @@ -143,5 +143,9 @@ def stream(key)
offset += chunk_size
end
end

def custom_metadata_headers(metadata)
metadata.transform_keys { |key| "x-amz-meta-#{key}" }
end
end
end
14 changes: 10 additions & 4 deletions activestorage/test/controllers/direct_uploads_controller_test.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,10 @@ class ActiveStorage::S3DirectUploadsControllerTest < ActionDispatch::Integration
"my_key_1": "my_value_1",
"my_key_2": "my_value_2",
"platform": "my_platform",
"library_ID": "12345"
"library_ID": "12345",
custom: {
"my_key_3": "my_value_3"
}
}

ActiveStorage::DirectUploadToken.stub(:verify_direct_upload_token, "s3") do
Expand All @@ -39,7 +42,7 @@ class ActiveStorage::S3DirectUploadsControllerTest < ActionDispatch::Integration
assert_equal "text/plain", details["content_type"]
assert_match SERVICE_CONFIGURATIONS[:s3][:bucket], details["direct_upload"]["url"]
assert_match(/s3(-[-a-z0-9]+)?\.(\S+)?amazonaws\.com/, details["direct_upload"]["url"])
assert_equal({ "Content-Type" => "text/plain", "Content-MD5" => checksum, "Content-Disposition" => "inline; filename=\"hello.txt\"; filename*=UTF-8''hello.txt" }, details["direct_upload"]["headers"])
assert_equal({ "Content-Type" => "text/plain", "Content-MD5" => checksum, "Content-Disposition" => "inline; filename=\"hello.txt\"; filename*=UTF-8''hello.txt", "x-amz-meta-my_key_3" => "my_value_3" }, details["direct_upload"]["headers"])
end
end
end
Expand Down Expand Up @@ -67,7 +70,10 @@ class ActiveStorage::GCSDirectUploadsControllerTest < ActionDispatch::Integratio
"my_key_1": "my_value_1",
"my_key_2": "my_value_2",
"platform": "my_platform",
"library_ID": "12345"
"library_ID": "12345",
custom: {
"my_key_3": "my_value_3"
}
}

ActiveStorage::DirectUploadToken.stub(:verify_direct_upload_token, "gcs") do
Expand All @@ -83,7 +89,7 @@ class ActiveStorage::GCSDirectUploadsControllerTest < ActionDispatch::Integratio
assert_equal metadata, details["metadata"].transform_keys(&:to_sym)
assert_equal "text/plain", details["content_type"]
assert_match %r{storage\.googleapis\.com/#{@config[:bucket]}}, details["direct_upload"]["url"]
assert_equal({ "Content-MD5" => checksum, "Content-Disposition" => "inline; filename=\"hello.txt\"; filename*=UTF-8''hello.txt" }, details["direct_upload"]["headers"])
assert_equal({ "Content-MD5" => checksum, "Content-Disposition" => "inline; filename=\"hello.txt\"; filename*=UTF-8''hello.txt", "x-goog-meta-my_key_3" => "my_value_3" }, details["direct_upload"]["headers"])
end
end
end
Expand Down
20 changes: 19 additions & 1 deletion activestorage/test/models/blob_test.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -259,13 +259,31 @@ class ActiveStorage::BlobTest < ActiveSupport::TestCase
test "updating the content_type updates service metadata" do
blob = directly_upload_file_blob(filename: "racecar.jpg", content_type: "application/octet-stream")

expected_arguments = [blob.key, content_type: "image/jpeg"]
expected_arguments = [blob.key, content_type: "image/jpeg", custom_metadata: {}]

assert_called_with(blob.service, :update_metadata, expected_arguments) do
blob.update!(content_type: "image/jpeg")
end
end

test "updating the metadata updates service metadata" do
blob = directly_upload_file_blob(filename: "racecar.jpg", content_type: "application/octet-stream")

expected_arguments = [
blob.key,
{
content_type: "application/octet-stream",
disposition: :attachment,
filename: blob.filename,
custom_metadatata: { "test" => true }
}
]

assert_called_with(blob.service, :update_metadata, expected_arguments) do
blob.update!(metadata: { custom: { "test" => true } })
end
end

test "scope_for_strict_loading adds includes only when track_variants and strict_loading_by_default" do
assert_empty(
ActiveStorage::Blob.scope_for_strict_loading.includes_values,
Expand Down
13 changes: 13 additions & 0 deletions activestorage/test/service/azure_storage_service_test.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,19 @@ class ActiveStorage::Service::AzureStorageServiceTest < ActiveSupport::TestCase
@service.delete key
end

test "upload with custom_metadata" do
key = SecureRandom.base58(24)
data = "Foobar"

@service.upload(key, StringIO.new(data), checksum: OpenSSL::Digest::MD5.base64digest(data), filename: ActiveStorage::Filename.new("test.txt"), custom_metadata: { "foo" => "baz" })
url = @service.url(key, expires_in: 2.minutes, disposition: :inline, content_type: "text/html", filename: ActiveStorage::Filename.new("test.html"))

response = Net::HTTP.get_response(URI(url))
assert_equal("baz", response["x-ms-meta-foo"])
ensure
@service.delete key
end

test "signed URL generation" do
url = @service.url(@key, expires_in: 5.minutes,
disposition: :inline, filename: ActiveStorage::Filename.new("avatar.png"), content_type: "image/png")
Expand Down
Loading

0 comments on commit e106a4a

Please sign in to comment.