lib/nn.rb in nn-2.0.1 vs lib/nn.rb in nn-2.1.0

@@ -1,10 +1,10 @@
 require "numo/narray"
 require "json"
 
 class NN
-  VERSION = "2.0"
+  VERSION = "2.1"
 
   include Numo
 
   attr_accessor :weights
   attr_accessor :biases
@@ -42,10 +42,14 @@
     @training = true
     init_layers
   end
 
   def self.load(file_name)
+    Marshal.load(File.binread(file_name))
+  end
+
+  def self.load_json(file_name)
     json = JSON.parse(File.read(file_name))
     nn = self.new(json["num_nodes"],
       learning_rate: json["learning_rate"],
       batch_size: json["batch_size"],
       activation: json["activation"].map(&:to_sym),
@@ -148,10 +152,14 @@
       forward(x, false)
     end
   end
 
   def save(file_name)
+    File.binwrite(file_name, Marshal.dump(self))
+  end
+
+  def save_json(file_name)
     json = {
       "version" => VERSION,
       "num_nodes" => @num_nodes,
       "learning_rate" => @learning_rate,
       "batch_size" => @batch_size,
@@ -238,12 +246,12 @@
     end
   end
 
   def update_weight_and_bias
     @layers.select{|layer| layer.is_a?(Affine)}.each.with_index do |layer, i|
-      weight_amount = layer.d_weight.mean(0) * @learning_rate
-      bias_amount = layer.d_bias.mean * @learning_rate
+      weight_amount = layer.d_weight * @learning_rate
+      bias_amount = layer.d_bias * @learning_rate
       if @momentum > 0
         weight_amount += @momentum * @weight_amounts[i]
         @weight_amounts[i] = weight_amount
         bias_amount += @momentum * @bias_amounts[i]
         @bias_amounts[i] = bias_amount
@@ -253,12 +261,12 @@
     end
   end
 
   def update_gamma_and_beta
     @layers.select{|layer| layer.is_a?(BatchNorm)}.each.with_index do |layer, i|
-      gamma_amount = layer.d_gamma.mean * @learning_rate
-      beta_amount = layer.d_beta.mean * @learning_rate
+      gamma_amount = layer.d_gamma * @learning_rate
+      beta_amount = layer.d_beta * @learning_rate
       if @momentum > 0
         gamma_amount += @momentum * @gamma_amounts[i]
         @gamma_amounts[i] = gamma_amount
         beta_amount += @momentum * @beta_amounts[i]
         @beta_amounts[i] = beta_amount
@@ -288,24 +296,26 @@
     @x.dot(@nn.weights[@index]) + @nn.biases[@index]
   end
 
   def backward(dout)
     x = @x.reshape(*@x.shape, 1)
-    @d_weight = x.dot(dout.reshape(dout.shape[0], 1, dout.shape[1]))
+    @d_weight = x.dot(dout.reshape(dout.shape[0], 1, dout.shape[1])).mean(0)
     if @nn.weight_decay > 0
       dridge = @nn.weight_decay * @nn.weights[@index]
       @d_weight += dridge
     end
-    @d_bias = dout
+    @d_bias = dout.mean
     dout.dot(@nn.weights[@index].transpose)
   end
 end
 
 
 class NN::Sigmoid
+  include Numo
+
   def forward(x)
-    @out = 1.0 / (1 + Numo::NMath.exp(-x))
+    @out = 1.0 / (1 + NMath.exp(-x))
   end
 
   def backward(dout)
     dout * (1.0 - @out) * @out
   end
@@ -326,12 +336,10 @@
   end
 end
 
 
 class NN::Identity
-  include Numo
-
   def initialize(nn)
     @nn = nn
   end
 
   def forward(x)
@@ -417,11 +425,11 @@
     out = @nn.gammas[@index] * @xn + @nn.betas[@index]
     out.reshape(*@x.shape)
   end
 
   def backward(dout)
-    @d_beta = dout.sum(0)
-    @d_gamma = (@xn * dout).sum(0)
+    @d_beta = dout.sum(0).mean
+    @d_gamma = (@xn * dout).sum(0).mean
     dxn = @nn.gammas[@index] * dout
     dxc = dxn / @std
     dstd = -((dxn * @xc) / (@std ** 2)).sum(0)
     dvar = 0.5 * dstd / @std
     dxc += (2.0 / @nn.batch_size) * @xc * dvar