updates to s3 support with remote tests

git-svn-id: http://svn.techno-weenie.net/projects/plugins/attachment_fu@2675 567b1171-46fb-0310-a4c9-b4bef9110e78
This commit is contained in:
technoweenie 2007-01-14 18:35:00 +00:00
Родитель 4b76298835
Коммит 7be8acf8c4
8 изменённых файлов: 320 добавлений и 75 удалений

Просмотреть файл

@ -1,14 +1,14 @@
development:
secret_access_key: AbCDEfGHiJKlmNOPQRS1
access_key_id: 1234567891abcdeFGHI/JKL+MnoPQrsT123UvwX4
bucket_prefix: appname_development
bucket_name: appname_development
access_key_id:
secret_access_key:
test:
secret_access_key: AbCDEfGHiJKlmNOPQRS1
access_key_id: 1234567891abcdeFGHI/JKL+MnoPQrsT123UvwX4
bucket_prefix: appname_test
bucket_name: appname_test
access_key_id:
secret_access_key:
production:
secret_access_key: AbCDEfGHiJKlmNOPQRS1
access_key_id: 1234567891abcdeFGHI/JKL+MnoPQrsT123UvwX4
bucket_prefix: appname
bucket_name: appname
access_key_id:
secret_access_key:

Просмотреть файл

@ -18,8 +18,8 @@ module Technoweenie # :nodoc:
# * <tt>:resize_to</tt> - Used by RMagick to resize images. Pass either an array of width/height, or a geometry string.
# * <tt>:thumbnails</tt> - Specifies a set of thumbnails to generate. This accepts a hash of filename suffixes and RMagick resizing options.
# * <tt>:thumbnail_class</tt> - Set what class to use for thumbnails. This attachment class is used by default.
# * <tt>:file_system_path</tt> - path to store the uploaded files. Uses public/#{table_name} by default.
# Setting this sets the :storage to :file_system.
# * <tt>:path_prefix</tt> - path to store the uploaded files. Uses public/#{table_name} by default for the filesystem, and just #{table_name}
# for the S3 backend. Setting this sets the :storage to :file_system.
# * <tt>:storage</tt> - Use :file_system to specify the attachment data is stored with the file system. Defaults to :db_system.
#
# Examples:
@ -30,10 +30,10 @@ module Technoweenie # :nodoc:
# has_attachment :content_type => :image, :resize_to => [50,50]
# has_attachment :content_type => ['application/pdf', :image], :resize_to => 'x50'
# has_attachment :thumbnails => { :thumb => [50, 50], :geometry => 'x50' }
# has_attachment :storage => :file_system, :file_system_path => 'public/files'
# has_attachment :storage => :file_system, :file_system_path => 'public/files',
# has_attachment :storage => :file_system, :path_prefix => 'public/files'
# has_attachment :storage => :file_system, :path_prefix => 'public/files',
# :content_type => :image, :resize_to => [50,50]
# has_attachment :storage => :file_system, :file_system_path => 'public/files',
# has_attachment :storage => :file_system, :path_prefix => 'public/files',
# :thumbnails => { :thumb => [50, 50], :geometry => 'x50' }
# has_attachment :storage => :s3
def has_attachment(options = {})
@ -50,9 +50,12 @@ module Technoweenie # :nodoc:
class_inheritable_accessor :attachment_options
attr_accessor :thumbnail_resize_options
options[:storage] ||= options[:file_system_path] ? :file_system : :db_file
options[:file_system_path] ||= File.join("public", table_name)
options[:file_system_path] = options[:file_system_path][1..-1] if options[:file_system_path].first == '/'
options[:storage] ||= (options[:file_system_path] || options[:path_prefix]) ? :file_system : :db_file
options[:path_prefix] ||= options[:file_system_path]
if options[:path_prefix].nil?
options[:path_prefix] = options[:storage] == :s3 ? table_name : File.join("public", table_name)
end
options[:path_prefix] = options[:path_prefix][1..-1] if options[:path_prefix].first == '/'
with_options :foreign_key => 'parent_id' do |m|
m.has_many :thumbnails, :dependent => :destroy, :class_name => options[:thumbnail_class].to_s

Просмотреть файл

@ -16,7 +16,7 @@ module Technoweenie # :nodoc:
# Overwrite this method in your model to customize the filename.
# The optional thumbnail argument will output the thumbnail's filename.
def full_filename(thumbnail = nil)
file_system_path = (thumbnail ? thumbnail_class : self).attachment_options[:file_system_path].to_s
file_system_path = (thumbnail ? thumbnail_class : self).attachment_options[:path_prefix].to_s
File.join(RAILS_ROOT, file_system_path, attachment_path_id, thumbnail_name_for(thumbnail))
end

Просмотреть файл

@ -3,50 +3,58 @@ module Technoweenie # :nodoc:
module Backends
# = AWS::S3 Storage Backend
#
# Enables use of Amazon's Simple Storage Service (http://aws.amazon.com/s3) as a storage mechanism
# Enables use of {Amazon's Simple Storage Service}[http://aws.amazon.com/s3] as a storage mechanism
#
# == Requirements
#
# Requires the AWS::S3 Library for S3 by Marcel Molina Jr. (http://amazon.rubyforge.org) installed either
# Requires the {AWS::S3 Library}[http://amazon.rubyforge.org] for S3 by Marcel Molina Jr. installed either
# as a gem or a as a Rails plugin.
#
# == Configuration
#
# Configuration is done via <tt>RAILS_ROOT/config/amazon_s3.yml</tt> and is loaded according to the <tt>RAILS_ENV</tt>.
# The minimum connection options that you must specify are your access key id and your secret access key.
# The minimum connection options that you must specify are a bucket name, your access key id and your secret access key.
# If you don't already have your access keys, all you need to sign up for the S3 service is an account at Amazon.
# You can sign up for S3 and get access keys by visiting http://aws.amazon.com/s3.
#
# Example configuration (RAILS_ROOT/config/amazon_s3.yml)
#
# development:
# secret_access_key: AbCDEfGHiJKlmNOPQRS1
# access_key_id: 1234567891abcdeFGHI/JKL+MnoPQrsT123UvwX4
# bucket_prefix: appname_development
# bucket_name: appname_development
# access_key_id: <your key>
# secret_access_key: <your key>
#
# test:
# secret_access_key: AbCDEfGHiJKlmNOPQRS1
# access_key_id: 1234567891abcdeFGHI/JKL+MnoPQrsT123UvwX4
# bucket_prefix: appname_test
# bucket_name: appname_test
# access_key_id: <your key>
# secret_access_key: <your key>
#
# production:
# secret_access_key: AbCDEfGHiJKlmNOPQRS1
# access_key_id: 1234567891abcdeFGHI/JKL+MnoPQrsT123UvwX4
# bucket_prefix: appname
# bucket_name: appname
# access_key_id: <your key>
# secret_access_key: <your key>
#
# === Required arguments
# === Required configuration parameters
#
# * <tt>:access_key_id</tt> - The access key id for your S3 account. Provided by Amazon.
# * <tt>:secret_access_key</tt> - The secret access key for your S3 account. Provided by Amazon.
# * <tt>:bucket_prefix</tt> - The string prefix to assign to each bucket. Used to create unique bucket names in the format <tt>#{bucket_prefix}_#{table_name}</tt>.
# * <tt>:bucket_name</tt> - A unique bucket name (think of the bucket_name as being like a database name).
#
# If any of these required arguments is missing, a MissingAccessKey exception will be raised from AWS::S3.
#
# === Optional arguments
# == About bucket names
#
# * <tt>:server</tt> - The server to make requests to. You can use this to specify your bucket in the subdomain, or your own domain's cname if you are using virtual hosted buckets. Defaults to <tt>s3.amazonaws.com</tt>.
# Bucket names have to be globaly unique across the S3 system. And you can only have up to 100 of them,
# so it's a good idea to think of a bucket as being like a database, hence the correspondance in this
# implementation to the development, test, and production environments.
#
# The number of objects you can store in a bucket is, for all intents and purposes, unlimited.
#
# === Optional configuration parameters
#
# * <tt>:server</tt> - The server to make requests to. Defaults to <tt>s3.amazonaws.com</tt>.
# * <tt>:port</tt> - The port to the requests should be made on. Defaults to 80 or 443 if <tt>:use_ssl</tt> is set.
# * <tt>:use_ssl</tt> - Whether requests should be made over SSL. If set to true, <tt>:port</tt> will be implicitly set to 443, unless specified otherwise. Defaults to false.
# * <tt>:use_ssl</tt> - If set to true, <tt>:port</tt> will be implicitly set to 443, unless specified otherwise. Defaults to false.
#
# == Usage
#
@ -56,79 +64,200 @@ module Technoweenie # :nodoc:
# has_attachment :storage => :s3
# end
#
# Of course, all the usual configuration options apply:
# === Customizing the path
#
# By default, files are prefixed using a pseudo hierarchy in the form of <tt>:table_name/:id</tt>, which results
# in S3 urls that look like: http(s)://:server/:bucket_name/:table_name/:id/:filename with :table_name
# representing the customizable portion of the path. You can customize this prefix using the <tt>:path_prefix</tt>
# option:
#
# class Photo < ActiveRecord::Base
# has_attachment :storage => :s3, :path_prefix => 'my/custom/path'
# end
#
# Which would result in URLs like <tt>http(s)://:server/:bucket_name/my/custom/path/:id/:filename.</tt>
#
# === Permissions
#
# By default, files are stored on S3 with public access permissions. You can customize this using
# the <tt>:s3_access</tt> option to <tt>has_attachment</tt>. Available values are
# <tt>:private</tt>, <tt>:public_read_write</tt>, and <tt>:authenticated_read</tt>.
#
# === Other options
#
# Of course, all the usual configuration options apply, such as content_type and thumbnails:
#
# class Photo < ActiveRecord::Base
# has_attachment :storage => :s3, :content_type => ['application/pdf', :image], :resize_to => 'x50'
# has_attachment :storage => :s3, :thumbnails => { :thumb => [50, 50], :geometry => 'x50' }
# end
#
# === Accessing S3 URLs
#
# You can get an object's URL using the s3_url accessor. For example, assuming that for your postcard app
# you had a bucket name like 'postcard_world_development', and an attachment model called Photo:
#
# @postcard.s3_url # => http(s)://s3.amazonaws.com/postcard_world_development/photos/1/mexico.jpg
#
# The resulting url is in the form: http(s)://:server/:bucket_name/:table_name/:id/:file.
# The optional thumbnail argument will output the thumbnail's filename (if any).
#
# Additionally, you can get an object's base path relative to the bucket root using
# <tt>base_path</tt>:
#
# @photo.file_base_path # => photos/1
#
# And the full path (including the filename) using <tt>full_filename</tt>:
#
# @photo.full_filename # => photos/
#
# Niether <tt>base_path</tt> or <tt>full_filename</tt> include the bucket name as part of the path.
# You can retrieve the bucket name using the <tt>bucket_name</tt> method.
module S3
class S3RequiredLibraryNotFound < StandardError; end
class S3ConfigFileNotFound < StandardError; end
class S3BucketExists < StandardError; end
class S3RequiredLibraryNotFoundError < StandardError; end
class S3ConfigFileNotFoundError < StandardError; end
def self.included(base) #:nodoc:
mattr_reader :bucket_name, :s3_config
begin
require 'aws/s3'
include AWS::S3
rescue LoadError
raise S3RequiredLibraryNotFound.new('AWS::S3 could not be loaded. Try installing with sudo gem i aws-s3, or see http://amazon.rubyforge.org for more information')
raise S3RequiredLibraryNotFoundError.new('AWS::S3 could not be loaded')
end
begin
@@s3_config = YAML.load_file(RAILS_ROOT + '/config/amazon_s3.yml')[ENV['RAILS_ENV']].symbolize_keys
rescue
raise S3ConfigFileNotFound.new('File RAILS_ROOT/config/amazon_s3.yml not found')
raise S3ConfigFileNotFoundError.new('File RAILS_ROOT/config/amazon_s3.yml not found')
end
@@bucket = [@@s3_config.delete(:bucket_prefix), base.table_name].join('_')
mattr_reader :s3_config, :bucket
@@bucket_name = s3_config[:bucket_name]
AWS::S3::Base.establish_connection!(s3_config)
find_or_create_bucket(bucket)
Base.establish_connection!(
:access_key_id => s3_config[:access_key_id],
:secret_access_key => s3_config[:secret_access_key],
:server => s3_config[:server],
:port => s3_config[:port],
:use_ssl => s3_config[:use_ssl]
)
# Bucket.create(@@bucket_name)
base.before_update :rename_file
end
def self.find_or_create_bucket(name)
AWS::S3::Bucket.find(name)
rescue AWS::S3::NoSuchBucket
AWS::S3::Bucket.create(name)
rescue AWS::S3::AccessDenied
raise S3BucketExists.new("Bucket name already exists: #{name}. Use a different bucket_prefix in RAILS_ROOT/config/amazon_s3.yml")
# Overwrites the base filename writer in order to store the old filename
def filename=(value)
@old_filename = filename unless filename.nil? || @old_filename
write_attribute :filename, sanitize_filename(value)
end
# Generates an S3 URL for the file in the form of: http(s)://<tt>{server}</tt>/<tt>{bucket_name}</tt>/<tt>{file_name}</tt>
# The <tt>{server}</tt> variable defaults to <tt>AWS::S3 URL::DEFAULT_HOST</tt> (http://s3.amazonaws.com) and can be
# set using the configuration parameters in <tt>RAILS_ROOT/config/amazon_s3.yml</tt>
# The attachment ID used in the full path of a file
def attachment_path_id
((respond_to?(:parent_id) && parent_id) || id).to_s
end
# The pseudo hierarchy containing the file relative to the bucket name
# Example: <tt>:table_name/:id</tt>
def base_path
File.join(attachment_options[:path_prefix], attachment_path_id)
end
# The full path to the file relative to the bucket name
# Example: <tt>:table_name/:id/:filename</tt>
def full_filename(thumbnail = nil)
File.join(base_path, thumbnail_name_for(thumbnail))
end
# All public objects are accessible via a GET request to the S3 servers. You can generate a
# url for an object using the s3_url method.
#
# Example usage: <tt>image_tag(@photo.s3_url)</tt>
# @photo.s3_url
#
# The resulting url is in the form: <tt>http(s)://:server/:bucket_name/:table_name/:id/:file</tt> where
# the <tt>:server</tt> variable defaults to <tt>AWS::S3 URL::DEFAULT_HOST</tt> (s3.amazonaws.com) and can be
# set using the configuration parameters in <tt>RAILS_ROOT/config/amazon_s3.yml</tt>.
#
# The optional thumbnail argument will output the thumbnail's filename (if any).
def s3_url(thumbnail = nil)
s3_config[:use_ssl] ? 'https://' : 'http://' + (s3_config[:server] || AWS::S3::DEFAULT_HOST) + '/' + bucket + '/' + thumbnail_name_for(thumbnail)
protocol, hostname = s3_config[:use_ssl] ? 'https://' : 'http://', s3_config[:server] || DEFAULT_HOST
protocol + File.join(hostname, bucket_name, full_filename(thumbnail))
end
alias :public_filename :s3_url
# All private objects are accessible via an authenticated GET request to the S3 servers. You can generate an
# authenticated url for an object like this:
#
# @photo.authenticated_s3_url
#
# By default authenticated urls expire 5 minutes after they were generated.
#
# Expiration options can be specified either with an absolute time using the <tt>:expires</tt> option,
# or with a number of seconds relative to now with the <tt>:expires_in</tt> option:
#
# # Absolute expiration date (October 13th, 2025)
# @photo.authenticated_s3_url(:expires => Time.mktime(2025,10,13).to_i)
#
# # Expiration in five hours from now
# @photo.authenticated_s3_url(:expires_in => 5.hours)
#
# You can specify whether the url should go over SSL with the <tt>:use_ssl</tt> option.
# By default, the ssl settings for the current connection will be used:
#
# @photo.authenticated_s3_url(:use_ssl => true)
#
# Finally, the optional thumbnail argument will output the thumbnail's filename (if any):
#
# @photo.authenticated_s3_url('thumbnail', :expires_in => 5.hours, :use_ssl => true)
def authenticated_s3_url(*args)
thumbnail = args.first.is_a?(String) ? args.first : nil
options = args.last.is_a?(Hash) ? args.last : {}
S3Object.url_for(full_filename(thumbnail), bucket_name, options)
end
def create_temp_file
write_to_temp_file current_data
end
def current_data
AWS::S3::S3Object.value filename, bucket
S3Object.value full_filename, bucket_name
end
protected
# Destroys the file. Called in the after_destroy callback
# Called in the after_destroy callback
def destroy_file
AWS::S3::S3Object.delete filename, bucket
S3Object.delete full_filename, bucket_name
end
def rename_file
return unless @old_filename && @old_filename != filename
AWS::S3::S3Object.rename(@old_filename, filename, bucket, :access => attachment_options[:s3_access])
old_full_filename = File.join(base_path, @old_filename)
S3Object.rename(
old_full_filename,
full_filename,
bucket_name,
:access => attachment_options[:s3_access]
)
@old_filename = nil
true
end
# Saves the file to S3
def save_to_storage
AWS::S3::S3Object.store(filename, temp_data, bucket, :content_type => content_type, :access => attachment_options[:s3_access]) if save_attachment?
if save_attachment?
S3Object.store(
full_filename,
temp_data,
bucket_name,
:content_type => content_type,
:access => attachment_options[:s3_access]
)
end
@old_filename = nil
true
end

Просмотреть файл

@ -0,0 +1,85 @@
require File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'test_helper'))
require 'net/http'
class S3Test < Test::Unit::TestCase
if File.exist?(File.dirname(__FILE__) + '/../../../../../../config/amazon_s3.yml')
include BaseAttachmentTests
attachment_model S3Attachment
def test_should_create_correct_bucket_name(klass = S3Attachment)
attachment_model klass
attachment = upload_file :filename => '/files/rails.png'
assert_equal attachment.s3_config[:bucket_name], attachment.bucket_name
end
test_against_subclass :test_should_create_correct_bucket_name, S3Attachment
def test_should_create_default_path_prefix(klass = S3Attachment)
attachment_model klass
attachment = upload_file :filename => '/files/rails.png'
assert_equal File.join(attachment_model.table_name, attachment.attachment_path_id), attachment.base_path
end
test_against_subclass :test_should_create_default_path_prefix, S3Attachment
def test_should_create_custom_path_prefix(klass = S3WithPathPrefixAttachment)
attachment_model klass
attachment = upload_file :filename => '/files/rails.png'
assert_equal File.join('some/custom/path/prefix', attachment.attachment_path_id), attachment.base_path
end
test_against_subclass :test_should_create_custom_path_prefix, S3WithPathPrefixAttachment
def test_should_create_valid_url(klass = S3Attachment)
attachment_model klass
attachment = upload_file :filename => '/files/rails.png'
assert_equal "http://s3.amazonaws.com/#{attachment.bucket_name}/#{attachment.full_filename}", attachment.s3_url
end
test_against_subclass :test_should_create_valid_url, S3Attachment
def test_should_create_authenticated_url(klass = S3Attachment)
attachment_model klass
attachment = upload_file :filename => '/files/rails.png'
assert_match /^http.+AWSAccessKeyId.+Expires.+Signature.+/, attachment.authenticated_s3_url(:use_ssl => true)
end
test_against_subclass :test_should_create_authenticated_url, S3Attachment
def test_should_save_attachment(klass = S3Attachment)
attachment_model klass
assert_created do
attachment = upload_file :filename => '/files/rails.png'
assert_valid attachment
assert attachment.image?
assert !attachment.size.zero?
assert_kind_of Net::HTTPOK, http_response_for(attachment.s3_url)
end
end
test_against_subclass :test_should_save_attachment, S3Attachment
def test_should_delete_attachment_from_s3_when_attachment_record_destroyed(klass = S3Attachment)
attachment_model klass
attachment = upload_file :filename => '/files/rails.png'
urls = [attachment.s3_url] + attachment.thumbnails.collect(&:s3_url)
urls.each {|url| assert_kind_of Net::HTTPOK, http_response_for(url) }
attachment.destroy
urls.each {|url| assert_kind_of Net::HTTPForbidden, http_response_for(url) }
end
test_against_subclass :test_should_delete_attachment_from_s3_when_attachment_record_destroyed, S3Attachment
protected
def http_response_for(url)
url = URI.parse(url)
Net::HTTP.start(url.host) {|http| http.request_head(url.path) }
end
else
def test_flunk_s3
puts "s3 config file not loaded, tests not running"
end
end
end

26
test/fixtures/attachment.rb поставляемый
Просмотреть файл

@ -40,17 +40,17 @@ class ImageWithThumbsAttachment < Attachment
end
class FileAttachment < ActiveRecord::Base
has_attachment :file_system_path => 'vendor/plugins/attachment_fu/test/files', :processor => :rmagick
has_attachment :path_prefix => 'vendor/plugins/attachment_fu/test/files', :processor => :rmagick
validates_as_attachment
end
class ImageFileAttachment < FileAttachment
has_attachment :file_system_path => 'vendor/plugins/attachment_fu/test/files',
has_attachment :path_prefix => 'vendor/plugins/attachment_fu/test/files',
:content_type => :image, :resize_to => [50,50]
end
class ImageWithThumbsFileAttachment < FileAttachment
has_attachment :file_system_path => 'vendor/plugins/attachment_fu/test/files',
has_attachment :path_prefix => 'vendor/plugins/attachment_fu/test/files',
:thumbnails => { :thumb => [50, 50], :geometry => 'x50' }, :resize_to => [55,55]
after_resize do |record, img|
record.aspect_ratio = img.columns.to_f / img.rows.to_f
@ -58,13 +58,14 @@ class ImageWithThumbsFileAttachment < FileAttachment
end
class ImageWithThumbsClassFileAttachment < FileAttachment
# use file_system_path to test backwards compatibility
has_attachment :file_system_path => 'vendor/plugins/attachment_fu/test/files',
:thumbnails => { :thumb => [50, 50] }, :resize_to => [55,55],
:thumbnail_class => 'ImageThumbnail'
end
class ImageThumbnail < FileAttachment
has_attachment :file_system_path => 'vendor/plugins/attachment_fu/test/files/thumbnails'
has_attachment :path_prefix => 'vendor/plugins/attachment_fu/test/files/thumbnails'
end
# no parent
@ -75,7 +76,7 @@ end
# no filename, no size, no content_type
class MinimalAttachment < ActiveRecord::Base
has_attachment :file_system_path => 'vendor/plugins/attachment_fu/test/files', :processor => :rmagick
has_attachment :path_prefix => 'vendor/plugins/attachment_fu/test/files', :processor => :rmagick
validates_as_attachment
def filename
@ -85,8 +86,21 @@ end
begin
class ImageScienceAttachment < ActiveRecord::Base
has_attachment :file_system_path => 'vendor/plugins/attachment_fu/test/files',
has_attachment :path_prefix => 'vendor/plugins/attachment_fu/test/files',
:processor => :image_science, :thumbnails => { :thumb => [50, 51], :geometry => '31>' }, :resize_to => 55
end
rescue MissingSourceFile
end
begin
class S3Attachment < ActiveRecord::Base
has_attachment :storage => :s3, :processor => :rmagick
validates_as_attachment
end
class S3WithPathPrefixAttachment < S3Attachment
has_attachment :storage => :s3, :path_prefix => 'some/custom/path/prefix', :processor => :rmagick
validates_as_attachment
end
rescue Technoweenie::AttachmentFu::Backends::S3::S3ConfigFileNotFoundError
end

Просмотреть файл

@ -49,4 +49,16 @@ ActiveRecord::Schema.define(:version => 0) do
create_table :db_files, :force => true do |t|
t.column :data, :binary
end
create_table :s3_attachments, :force => true do |t|
t.column :parent_id, :integer
t.column :thumbnail, :string
t.column :filename, :string, :limit => 255
t.column :content_type, :string, :limit => 255
t.column :size, :integer
t.column :width, :integer
t.column :height, :integer
t.column :type, :string
t.column :aspect_ratio, :float
end
end

Просмотреть файл

@ -1,5 +1,7 @@
$:.unshift(File.dirname(__FILE__) + '/../lib')
ENV['RAILS_ENV'] = 'test'
require 'test/unit'
require File.expand_path(File.join(File.dirname(__FILE__), '../../../../config/environment.rb'))
require 'breakpoint'