layers.rb in ruby-dnn-0.9.0

- old
+ new

@@ -1,53 +1,47 @@
 module DNN
   module Layers
 
     # Super class of all optimizer classes.
     class Layer
+      attr_reader :input_shape
+
       def initialize
         @built = false
       end
 
       # Build the layer.
-      def build(model)
-        @model = model
+      def build(input_shape)
+        @input_shape = input_shape
         @built = true
       end
       
       # Does the layer have already been built?
       def built?
         @built
       end
 
       # Forward propagation.
-      # Classes that inherit from this class must implement this method.
       def forward(x)
         raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'forward'")
       end
 
       # Backward propagation.
-      # Classes that inherit from this class must implement this method.
       def backward(dout)
         raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'update'")
       end
-    
-      # Get the shape of the layer.
-      def shape
-        prev_layer.shape
+
+      def output_shape
+        @input_shape
       end
 
       # Layer to a hash.
       def to_hash(merge_hash = nil)
         hash = {class: self.class.name}
         hash.merge!(merge_hash) if merge_hash
         hash
       end
-    
-      # Get the previous layer.
-      def prev_layer
-        @model.get_prev_layer(self)
-      end
     end
     
     
     # This class is a superclass of all classes with learning parameters.
     class HasParamLayer < Layer
@@ -58,63 +52,67 @@
         super()
         @params = {}
         @trainable = true
       end
     
-      def build(model)
-        @model = model
+      def build(input_shape)
+        @input_shape = input_shape
         unless @built
           @built = true
           init_params
         end
       end
     
       # Update the parameters.
-      def update
-        @model.optimizer.update(@params) if @trainable
+      def update(optimizer)
+        optimizer.update(@params) if @trainable
       end
     
       private
       
       # Initialize of the parameters.
-      # Classes that inherit from this class must implement this method.
       def init_params
         raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'init_params'")
       end
     end
     
     
     class InputLayer < Layer
-      attr_reader :shape
-
       def self.load_hash(hash)
-        self.new(hash[:shape])
+        self.new(hash[:input_shape])
       end
 
-      def initialize(dim_or_shape)
+      def initialize(input_dim_or_shape)
         super()
-        @shape = dim_or_shape.is_a?(Array) ? dim_or_shape : [dim_or_shape]
+        @input_shape = input_dim_or_shape.is_a?(Array) ? input_dim_or_shape : [input_dim_or_shape]
       end
 
+      def build
+        @built = true
+        @input_shape
+      end
+
       def forward(x)
         x
       end
     
       def backward(dout)
         dout
       end
 
       def to_hash
-        super({shape: @shape})
+        super({input_shape: @input_shape})
       end
     end
 
 
     # It is a superclass of all connection layers.
     class Connection < HasParamLayer
       attr_reader :l1_lambda # L1 regularization
       attr_reader :l2_lambda # L2 regularization
+      attr_reader :weight_initializer
+      attr_reader :bias_initializer
 
       def initialize(weight_initializer: Initializers::RandomNormal.new,
                      bias_initializer: Initializers::Zeros.new,
                      l1_lambda: 0,
                      l2_lambda: 0)
@@ -141,19 +139,19 @@
         else
           0
         end
       end
 
-      def dlasso
+      def d_lasso
         if @l1_lambda > 0
           dlasso = Xumo::SFloat.ones(*@weight.data.shape)
           dlasso[@weight.data < 0] = -1
           @weight.grad += @l1_lambda * dlasso
         end
       end
 
-      def dridge
+      def d_ridge
         if @l2_lambda > 0
           @weight.grad += @l2_lambda * @weight.data
         end
       end
 
@@ -176,12 +174,12 @@
     class Dense < Connection
       attr_reader :num_nodes
 
       def self.load_hash(hash)
         self.new(hash[:num_nodes],
-                 weight_initializer: Util.load_hash(hash[:weight_initializer]),
-                 bias_initializer: Util.load_hash(hash[:bias_initializer]),
+                 weight_initializer: Utils.load_hash(hash[:weight_initializer]),
+                 bias_initializer: Utils.load_hash(hash[:bias_initializer]),
                  l1_lambda: hash[:l1_lambda],
                  l2_lambda: hash[:l2_lambda])
       end
     
       def initialize(num_nodes,
@@ -203,130 +201,104 @@
         @weight.grad = @x.transpose.dot(dout)
         @bias.grad = dout.sum(0)
         dout.dot(@weight.data.transpose)
       end
     
-      def shape
+      def output_shape
         [@num_nodes]
       end
 
       def to_hash
         super({num_nodes: @num_nodes})
       end
     
       private
     
       def init_params
-        num_prev_nodes = prev_layer.shape[0]
+        num_prev_nodes = @input_shape[0]
         @weight.data = Xumo::SFloat.new(num_prev_nodes, @num_nodes)
         @bias.data = Xumo::SFloat.new(@num_nodes)
         super()
       end
     end
     
 
     class Flatten < Layer
       def forward(x)
-        @shape = x.shape
-        x.reshape(x.shape[0], x.shape[1..-1].reduce(:*))
+        x.reshape(x.shape[0], *output_shape)
       end
     
       def backward(dout)
-        dout.reshape(*@shape)
+        dout.reshape(dout.shape[0], *@input_shape)
       end
-    
-      def shape
-        [prev_layer.shape.reduce(:*)]
+
+      def output_shape
+        [@input_shape.reduce(:*)]
       end
     end
 
 
     class Reshape < Layer
-      attr_reader :shape
-      
-      def initialize(shape)
-        super()
-        @shape = shape
-        @x_shape = nil
+      def self.load_hash(hash)
+        self.new(hash[:output_shape])
       end
 
-      def self.load_hash(hash)
-        self.new(hash[:shape])
+      def initialize(output_shape)
+        super()
+        @output_shape = output_shape
       end
 
       def forward(x)
-        @x_shape = x.shape
-        x.reshape(x.shape[0], *@shape)
+        x.reshape(x.shape[0], *@output_shape)
       end
 
       def backward(dout)
-        dout.reshape(*@x_shape)
+        dout.reshape(dout.shape[0], *@input_shape)
       end
 
+      def output_shape
+        @output_shape
+      end
+
       def to_hash
-        super({shape: @shape})
+        super({output_shape: @output_shape})
       end
     end
 
-
-    class OutputLayer < Layer
-      # Classes that inherit from this class must implement this method.
-      def loss(x)
-        raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'forward'")
-      end
-
-      def dloss
-        @model.get_all_layers.select { |layer| layer.is_a?(Connection) }.each do |layer|
-          layer.dlasso
-          layer.dridge
-        end
-      end
-      
-      private
-
-      def lasso
-        @model.get_all_layers.select { |layer| layer.is_a?(Connection) }
-                             .reduce(0) { |sum, layer| sum + layer.lasso }
-      end
     
-      def ridge
-        @model.get_all_layers.select { |layer| layer.is_a?(Connection) }
-                             .reduce(0) { |sum, layer| sum + layer.ridge }
-      end
-    end
-    
-    
     class Dropout < Layer
       attr_reader :dropout_ratio
 
       def self.load_hash(hash)
-        self.new(hash[:dropout_ratio])
+        self.new(hash[:dropout_ratio], hash[:seed])
       end
 
-      def initialize(dropout_ratio = 0.5)
+      def initialize(dropout_ratio = 0.5, seed = rand(1 << 31))
         super()
         @dropout_ratio = dropout_ratio
+        @seed = seed
         @mask = nil
       end
-    
-      def forward(x)
-        if @model.training?
+
+      def forward(x, learning_phase)
+        if learning_phase
+          Xumo::SFloat.srand(@seed)
           @mask = Xumo::SFloat.ones(*x.shape).rand < @dropout_ratio
           x[@mask] = 0
         else
           x *= (1 - @dropout_ratio)
         end
         x
       end
     
-      def backward(dout)
-        dout[@mask] = 0 if @model.training?
+      def backward(dout, learning_phase)
+        dout[@mask] = 0 if learning_phase
         dout
       end
 
       def to_hash
-        super({dropout_ratio: @dropout_ratio})
+        super({dropout_ratio: @dropout_ratio, seed: @seed})
       end
     end
     
     
     class BatchNormalization < HasParamLayer
@@ -339,12 +311,12 @@
       def initialize(momentum: 0.9)
         super()
         @momentum = momentum
       end
 
-      def forward(x)
-        if @model.training?
+      def forward(x, learning_phase)
+        if learning_phase
           mean = x.mean(0)
           @xc = x - mean
           var = (@xc**2).mean(0)
           @std = Xumo::NMath.sqrt(var + 1e-7)
           xn = @xc / @std
@@ -356,11 +328,11 @@
           xn = xc / Xumo::NMath.sqrt(@running_var.data + 1e-7)
         end
         @gamma.data * xn + @beta.data
       end
     
-      def backward(dout)
+      def backward(dout, learning_phase)
         batch_size = dout.shape[0]
         @beta.grad = dout.sum(0)
         @gamma.grad = (@xn * dout).sum(0)
         dxn = @gamma.data * dout
         dxc = dxn / @std
@@ -376,13 +348,13 @@
       end
     
       private
     
       def init_params
-        @params[:gamma] = @gamma = Param.new(Xumo::SFloat.ones(*shape))
-        @params[:beta] = @beta = Param.new(Xumo::SFloat.zeros(*shape))
-        @params[:running_mean] = @running_mean = Param.new(Xumo::SFloat.zeros(*shape))
-        @params[:running_var] = @running_var = Param.new(Xumo::SFloat.zeros(*shape))
+        @params[:gamma] = @gamma = Param.new(Xumo::SFloat.ones(*output_shape))
+        @params[:beta] = @beta = Param.new(Xumo::SFloat.zeros(*output_shape))
+        @params[:running_mean] = @running_mean = Param.new(Xumo::SFloat.zeros(*output_shape))
+        @params[:running_var] = @running_var = Param.new(Xumo::SFloat.zeros(*output_shape))
       end
     end
   end
   
 end