require 'find'
require 'time'
require 'set'
class Cloudinary::Static
IGNORE_FILES = [".svn", "CVS", "RCS", ".git", ".hg"]
DEFAULT_IMAGE_DIRS = ["app/assets/images", "lib/assets/images", "vendor/assets/images", "public/images"]
DEFAULT_IMAGE_EXTENSION_MASK = 'gif|jpe?g|png|bmp|ico|webp|wdp|jxr|jp2|svg|pdf'
METADATA_FILE = ".cloudinary.static"
METADATA_TRASH_FILE = ".cloudinary.static.trash"
class << self
def sync(options={})
options = options.clone
delete_missing = options.delete(:delete_missing)
found_paths = Set.new
found_public_paths = {}
found_public_ids = Set.new
metadata = build_metadata
metadata_lines = []
counts = { :not_changed => 0, :uploaded => 0, :deleted => 0, :not_found => 0}
discover_all do |path, public_path|
next if found_paths.include?(path)
if found_public_paths[public_path]
print "Warning: duplicate #{public_path} in #{path} - already taken from #{found_public_paths[public_path]}\n"
next
end
found_paths << path
found_public_paths[public_path] = path
data = root.join(path).read(:mode=>"rb")
ext = path.extname
format = ext[1..-1]
md5 = Digest::MD5.hexdigest(data)
public_id = "#{public_path.basename(ext)}-#{md5}"
found_public_ids << public_id
item_metadata = metadata.delete(public_path.to_s)
if item_metadata && item_metadata["public_id"] == public_id # Signature match
counts[:not_changed] += 1
print "#{public_path} - #{public_id} - Not changed\n"
result = item_metadata
else
counts[:uploaded] += 1
print "#{public_path} - #{public_id} - Uploading\n"
result = Cloudinary::Uploader.upload(Cloudinary::Blob.new(data, :original_filename=>path.to_s),
options.merge(:format=>format, :public_id=>public_id, :type=>:asset, :resource_type=>resource_type(path.to_s))
).merge("upload_time"=>Time.now)
end
metadata_lines << [public_path, public_id, result["upload_time"].to_i, result["version"], result["width"], result["height"]].join("\t")+"\n"
end
File.open(metadata_file_path, "w"){|f| f.print(metadata_lines.join)}
metadata.to_a.each do |path, info|
counts[:not_found] += 1
print "#{path} - #{info["public_id"]} - Not found\n"
end
# Files no longer needed
trash = metadata.to_a + build_metadata(metadata_trash_file_path, false).reject{|public_path, info| found_public_ids.include?(info["public_id"])}
if delete_missing
trash.each do
|path, info|
counts[:deleted] += 1
print "#{path} - #{info["public_id"]} - Deleting\n"
Cloudinary::Uploader.destroy(info["public_id"], options.merge(:type=>:asset))
end
FileUtils.rm_f(metadata_trash_file_path)
else
# Add current removed file to the trash file.
metadata_lines = trash.map do
|public_path, info|
[public_path, info["public_id"], info["upload_time"].to_i, info["version"], info["width"], info["height"]].join("\t")+"\n"
end
File.open(metadata_trash_file_path, "w"){|f| f.print(metadata_lines.join)}
end
print "\nCompleted syncing static resources to Cloudinary\n"
print counts.sort.reject{|k,v| v == 0}.map{|k,v| "#{v} #{k.to_s.gsub('_', ' ').capitalize}"}.join(", ") + "\n"
end
# ## Cloudinary::Utils support ###
def public_id_and_resource_type_from_path(path)
@metadata ||= build_metadata
path = path.sub(/^\//, '')
prefix = public_prefixes.find {|prefix| @metadata[File.join(prefix, path)]}
if prefix
[@metadata[File.join(prefix, path)]['public_id'], resource_type(path)]
else
nil
end
end
private
def root
Cloudinary.app_root
end
def metadata_file_path
root.join(METADATA_FILE)
end
def metadata_trash_file_path
root.join(METADATA_TRASH_FILE)
end
def build_metadata(metadata_file = metadata_file_path, hash = true)
metadata = []
if File.exist?(metadata_file)
IO.foreach(metadata_file) do
|line|
line.strip!
next if line.blank?
path, public_id, upload_time, version, width, height = line.split("\t")
metadata << [path, {
"public_id" => public_id,
"upload_time" => Time.at(upload_time.to_i).getutc,
"version" => version,
"width" => width.to_i,
"height" => height.to_i
}]
end
end
hash ? Hash[*metadata.flatten] : metadata
end
def discover_all(&block)
static_file_config.each do |group, data|
print "-> Syncing #{group}...\n"
discover(absolutize(data['dirs']), extension_matcher_for(group), &block)
print "=========================\n"
end
end
def discover(dirs, matcher)
return unless matcher
dirs.each do |dir|
print "Scanning #{dir.relative_path_from(root)}...\n"
dir.find do |path|
file = path.basename.to_s
if ignore_file?(file)
Find.prune
next
elsif path.directory? || !matcher.call(path.to_s)
next
else
relative_path = path.relative_path_from(root)
public_path = path.relative_path_from(dir.dirname)
yield(relative_path, public_path)
end
end
end
end
def ignore_file?(file)
matches?(file, Cloudinary.config.ignore_files || IGNORE_FILES)
end
# Test for matching either strings or regexps
def matches?(target, patterns)
Array(patterns).any? {|pattern| pattern.is_a?(String) ? pattern == target : target.match(pattern)}
end
def extension_matcher_for(group)
group = group.to_s
return unless static_file_config[group]
@matchers = {}
@matchers[group] ||= ->(target) do
!!target.match(extension_mask_to_regex(static_file_config[group]['file_mask']))
end
end
def static_file_config
@static_file_config ||= begin
config = Cloudinary.config.static_files || {}
# Default
config['images'] ||= {}
config['images']['dirs'] ||= Cloudinary.config.static_image_dirs # Backwards compatibility
config['images']['dirs'] ||= DEFAULT_IMAGE_DIRS
config['images']['file_mask'] ||= DEFAULT_IMAGE_EXTENSION_MASK
# Validate
config.each do |group, data|
unless data && data['dirs'] && data['file_mask']
print "In config, static_files group '#{group}' needs to have both 'dirs' and 'file_mask' defined.\n"
exit
end
end
config
end
end
def reset_static_file_config!
@static_file_config = nil
end
def image?(path)
extension_matcher_for(:images).call(path)
end
def resource_type(path)
if image?(path)
:image
else
:raw
end
end
def extension_mask_to_regex(extension_mask)
extension_mask && /\.(?:#{extension_mask})$/i
end
def public_prefixes
@public_prefixes ||= static_file_config.reduce([]) do |result, (group, data)|
result << data['dirs'].map { |dir| Pathname.new(dir).basename.to_s }
end.flatten.uniq
end
def absolutize(dirs)
dirs.map do |relative_dir|
absolute_dir = root.join(relative_dir)
if absolute_dir.exist?
absolute_dir
else
print "Skipping #{relative_dir} (does not exist)\n"
nil
end
end.compact
end
def print(s)
$stderr.print(s)
end
end
end