script/worker_analysis in image_optim-0.20.2 vs script/worker_analysis in image_optim-0.21.0

@@ -7,11 +7,12 @@
 require 'image_optim/cmd'
 require 'progress'
 require 'shellwords'
 require 'gdbm'
 require 'digest'
-require 'haml'
+require 'erb'
+require 'ostruct'
 
 DIR = 'tmp'
 Pathname(DIR).mkpath
 
 Array.class_eval do
@@ -86,11 +87,11 @@
 
   def digest
     @digest ||= Digest::SHA256.file(to_s).hexdigest
   end
 
-  def cache_etag
+  def etag
     [mtime, digest]
   end
 end
 
 # Analyse efficency of workers
@@ -98,64 +99,57 @@
   Cmd = ImageOptim::Cmd
   HashHelpers = ImageOptim::HashHelpers
 
   # Caching entries using GDBM
   class Cache
-    PATH = "#{DIR}/worker-analysis.db"
+    DB = GDBM.new("#{DIR}/worker-analysis.db")
 
     class << self
-      def get(key, etag, &block)
+      def get(context, key, etag, &block)
+        full_key = [context, key]
         if block
-          get!(key, etag) || set!(key, etag, &block)
+          get!(full_key, etag) || set!(full_key, etag, &block)
         else
-          get!(key, etag)
+          get!(full_key, etag)
         end
       end
 
-      def set(key, etag, &block)
-        set!(key, etag, &block)
+      def set(context, key, etag, &block)
+        set!([context, key], etag, &block)
       end
 
     private
 
-      def open
-        GDBM.open(PATH) do |db|
-          yield db
-        end
-      end
-
       def get!(key, etag)
-        raw = open{ |db| db[Marshal.dump(key)] }
+        raw = DB[Marshal.dump(key)]
         return unless raw
         entry = Marshal.load(raw)
         return unless entry[1] == etag
         entry[0]
       end
 
       def set!(key, etag, &block)
         value = block.call
-        open{ |db| db[Marshal.dump(key)] = Marshal.dump([value, etag]) }
+        DB[Marshal.dump(key)] = Marshal.dump([value, etag])
         value
       end
     end
   end
 
   # Delegate to worker with short id
   class WorkerVariant < DelegateClass(ImageOptim::Worker)
-    attr_reader :cons_id, :id
+    attr_reader :name, :id, :cons_id
     def initialize(klass, image_optim, options)
       allow_consecutive_on = Array(options.delete(:allow_consecutive_on))
       @image_optim = image_optim
-      @id = klass.bin_sym.to_s
-      unless options.empty?
-        @id << "(#{options.map{ |k, v| "#{k}:#{v.inspect}" }.join(', ')})"
-      end
+      @name = klass.bin_sym.to_s + options_string(options)
       __setobj__(klass.new(image_optim, options))
+      @id = klass.bin_sym.to_s + options_string(self.options)
       @cons_id = [klass, allow_consecutive_on.map{ |key| [key, send(key)] }]
     end
 
-    def cache_etag
+    def etag
       [
         id,
         bin_versions,
         source_digest,
       ]
@@ -173,32 +167,53 @@
       @digest ||= begin
         source_path = __getobj__.method(:optimize).source_location[0]
         Digest::SHA256.file(source_path).hexdigest
       end
     end
+
+    def options_string(options)
+      return '' if options.empty?
+      "(#{options.sort.map{ |k, v| "#{k}:#{v.inspect}" }.join(', ')})"
+    end
   end
 
   # One worker result
   StepResult = Struct.new(*[
     :worker_id,
     :success,
     :time,
     :src_size,
     :dst_size,
+    :cache,
   ]) do
-    def self.run(src, dst, worker)
+    def self.run(src, worker)
+      dst = src.temp_path
       start = Process.times.sum
       success = worker.optimize(src, dst)
       time = Process.times.sum - start
 
-      new(worker.id, success, time, src.size, success ? dst.size : nil)
+      dst_size = success ? dst.size : nil
+      cache = (success ? dst : src).digest.sub(/../, '\0/') + ".#{src.format}"
+      result = new(worker.id, success, time, src.size, dst_size, cache)
+      if success
+        path = result.path
+        unless path.exist?
+          path.dirname.mkpath
+          dst.rename(path)
+        end
+      end
+      result
     end
 
     def size
       success ? dst_size : src_size
     end
 
+    def path
+      ImageOptim::ImagePath.convert("#{DIR}/worker-analysis/#{cache}")
+    end
+
     def inspect
       "<S:#{worker_id} #{success ? '✓' : '✗'} #{time}s #{src_size}→#{dst_size}>"
     end
   end
 
@@ -239,17 +254,14 @@
       @path = ImageOptim::ImagePath.convert(path)
       @workers = workers
     end
 
     def results
-      cache_etag = [@path.cache_etag, @workers.map(&:cache_etag).sort]
-      Cache.get(@path.to_s, cache_etag) do
-        results = []
-        run_workers(@path, @workers){ |result| results << result }
-        run_cache.clear
-        results
-      end
+      results = []
+      run_workers(@path, @workers){ |result| results << result }
+      run_cache.clear
+      results
     end
 
   private
 
     def run_cache
@@ -272,25 +284,33 @@
         chain_result = ChainResult.new(src.format, steps)
         chain_result.difference = difference_with(result_image)
 
         block.call(chain_result)
 
-        workers_left = workers.reject{ |w| w.cons_id == worker.cons_id }
+        workers_left = workers.reject do |w|
+          w.cons_id == worker.cons_id || w.run_order < worker.run_order
+        end
         run_workers(result_image, workers_left, chain_result, &block)
       end
     end
 
     def run_worker(src, worker)
       run_cache[:run][[src.digest, worker.id]] ||= begin
-        dst = src.temp_path
-        worker_result = StepResult.run(src, dst, worker)
-        [worker_result, worker_result.success ? dst : src]
+        cache_args = [:result, [src.digest, worker.id], worker.etag]
+        result = Cache.get(*cache_args)
+        if !result || (result.success && !result.path.exist?)
+          result = Cache.set(*cache_args) do
+            StepResult.run(src, worker)
+          end
+        end
+        [result, result.success ? result.path : src]
       end
     end
 
     def difference_with(other)
-      run_cache[:difference][other.digest] ||= begin
+      run_cache[:difference][other.digest] ||=
+      Cache.get(:difference, [@path.digest, other.digest].sort, nil) do
         images = [flatten_animation(@path), flatten_animation(other)]
 
         alpha_presence = images.map do |image|
           Cmd.capture("identify -format '%A' #{image.shellescape}")
         end
@@ -363,34 +383,40 @@
       attr_reader :worker_stats
       attr_reader :unused_workers
       attr_reader :entry_count
       attr_reader :original_size, :optimized_size, :ratio, :avg_ratio
       attr_reader :avg_difference, :max_difference, :warn_level
-      attr_reader :time, :speed
+      attr_reader :time, :avg_time, :speed
 
-      def initialize(worker_ids, results)
-        steps_by_worker_id = results.flat_map(&:steps).group_by(&:worker_id)
-        @worker_stats = worker_ids.map do |worker_id|
-          Worker.new(worker_id, steps_by_worker_id[worker_id])
-        end
+      def initialize(worker_ids, results, ids2names)
+        @worker_stats = build_worker_stats(worker_ids, results, ids2names)
         @unused_workers = worker_stats.any?(&:unused?)
 
         @entry_count = results.count
         @original_size = results.sum(&:src_size)
         @optimized_size = results.sum(&:dst_size)
         @ratio = optimized_size.to_f / original_size
         @avg_ratio = results.sum(&:ratio) / results.length
         @avg_difference = results.sum(&:difference) / results.length
         @max_difference = results.map(&:difference).max
         @time = results.sum(&:time)
+        @avg_time = time / results.length
 
         @warn_level = calculate_warn_level
         @speed = calculate_speed
       end
 
     private
 
+      def build_worker_stats(worker_ids, results, ids2names)
+        steps_by_worker_id = results.flat_map(&:steps).group_by(&:worker_id)
+        worker_ids.map do |worker_id|
+          worker_name = ids2names[worker_id] || worker_id
+          Worker.new(worker_name, steps_by_worker_id[worker_id])
+        end
+      end
+
       def calculate_warn_level
         case
         when max_difference >= 0.1 then 'high'
         when max_difference >= 0.01 then 'medium'
         when max_difference >= 0.001 then 'low'
@@ -408,29 +434,33 @@
 
     # Worker usage
     class Worker
       attr_reader :name
       attr_reader :success_count
+      attr_reader :time, :avg_time
       def initialize(name, steps)
         @name = name
         @success_count = steps.count(&:success)
+        @time = steps.sum(&:time)
+        @avg_time = time / steps.length
       end
 
       def unused?
         success_count.zero?
       end
     end
 
-    attr_reader :name, :results
-    def initialize(name, results)
+    attr_reader :name, :results, :ids2names
+    def initialize(name, results, ids2names)
       @name = name.to_s
       @results = results
+      @ids2names = ids2names
     end
 
     def each_chain(&block)
       chains = results.group_by(&:worker_ids).map do |worker_ids, results|
-        Chain.new(worker_ids, results)
+        Chain.new(worker_ids, results, ids2names)
       end
       chains.sort_by!{ |chain| [chain.optimized_size, chain.time] }
       chains.each(&block)
     end
   end
@@ -453,47 +483,59 @@
       end
       worker_option_variants.each do |options|
         options = HashHelpers.deep_symbolise_keys(options)
         options[:allow_consecutive_on] = allow_consecutive_on
         worker = WorkerVariant.new(klass, image_optim, options)
-        puts worker.id
         worker.image_formats.each do |format|
           @workers_by_format[format] << worker
         end
       end
     end
 
+    log_workers_by_format
+
     fail "unknown variants: #{option_variants}" unless option_variants.empty?
   end
 
   def analyse(paths)
-    results = process_paths(paths).shuffle.with_progress.flat_map do |path|
-      WorkerRunner.new(path, workers_for_image(path)).results
-    end
+    results = collect_results(paths)
 
-    template = Haml::Engine.new(File.read("#{__FILE__}.haml"))
+    template = ERB.new(template_path.read, nil, '>')
     by_format = results.group_by(&:format)
     formats = by_format.keys.sort
     basenames = Hash[formats.map do |format|
       [format, "worker-analysis-#{format}.html"]
     end]
     formats.each do |format|
-      stats = Stats.new('all', by_format[format])
+      stats = Stats.new('all', by_format[format], worker_ids2names)
+      path = FSPath("#{DIR}/#{basenames[format]}")
       model = {
         :stats_format => format,
         :stats => stats,
         :format_links => basenames,
+        :template_dir => template_path.dirname.relative_path_from(path.dirname),
       }
-      html = template.render(nil, model)
-      path = FSPath("#{DIR}/#{basenames[format]}")
+      html = template.result(OpenStruct.new(model).instance_eval{ binding })
       path.write(html)
       puts "Created #{path}"
     end
   end
 
 private
 
+  def worker_ids2names
+    Hash[@workers_by_format.values.flatten.map do |worker|
+      [worker.id, worker.name]
+    end]
+  end
+
+  def collect_results(paths)
+    process_paths(paths).shuffle.with_progress.flat_map do |path|
+      WorkerRunner.new(path, workers_for_image(path)).results
+    end
+  end
+
   def process_paths(paths)
     paths = paths.map{ |path| ImageOptim::ImagePath.convert(path) }
     paths.select!{ |path| path.exist? || warn("#{path} doesn't exits") }
     paths.select!{ |path| path.file? || warn("#{path} is not a file") }
     paths.select!{ |path| path.format || warn("#{path} is not an image") }
@@ -503,9 +545,22 @@
     paths
   end
 
   def workers_for_image(path)
     @workers_by_format[ImageOptim::ImagePath.convert(path).format]
+  end
+
+  def log_workers_by_format
+    @workers_by_format.each do |format, workers|
+      puts "#{format}:"
+      workers.sort_by.with_index{ |w, i| [w.run_order, i] }.each do |worker|
+        puts "  #{worker.name} [#{worker.run_order}]"
+      end
+    end
+  end
+
+  def template_path
+    FSPath("#{File.dirname(__FILE__)}/template/#{File.basename(__FILE__)}.erb")
   end
 end
 
 def option_variants
   path = '.analysis_variants.yml'