lib/dnn/core/activations.rb in ruby-dnn-0.6.10 vs lib/dnn/core/activations.rb in ruby-dnn-0.7.0

@@ -1,22 +1,26 @@
 module DNN
   module Activations
 
     class Sigmoid < Layers::Layer
+      NMath = Xumo::NMath
+
       def forward(x)
-        @out = 1 / (1 + Xumo::NMath.exp(-x))
+        @out = 1 / (1 + NMath.exp(-x))
       end
     
       def backward(dout)
         dout * (1 - @out) * @out
       end
     end
 
 
     class Tanh < Layers::Layer
+      NMath = Xumo::NMath
+
       def forward(x)
-        @out = Xumo::NMath.tanh(x)
+        @out = NMath.tanh(x)
       end
       
       def backward(dout)
         dout * (1 - @out**2)
       end
@@ -34,29 +38,33 @@
       end
     end
 
 
     class Softplus < Layers::Layer
+      NMath = Xumo::NMath
+
       def forward(x)
         @x = x
-        Xumo::NMath.log(1 + Xumo::NMath.exp(x))
+        NMath.log(1 + NMath.exp(x))
       end
 
       def backward(dout)
-        dout * (1 / (1 + Xumo::NMath.exp(-@x)))
+        dout * (1 / (1 + NMath.exp(-@x)))
       end
     end
 
 
     class Swish < Layers::Layer
+      NMath = Xumo::NMath
+
       def forward(x)
         @x = x
-        @out = x * (1 / (1 + Xumo::NMath.exp(-x)))
+        @out = x * (1 / (1 + NMath.exp(-x)))
       end
     
       def backward(dout)
-        dout * (@out + (1 / (1 + Xumo::NMath.exp(-@x))) * (1 - @out))
+        dout * (@out + (1 / (1 + NMath.exp(-@x))) * (1 - @out))
       end
     end
     
     
     class ReLU < Layers::Layer
@@ -103,10 +111,12 @@
       end
     end
 
 
     class ELU < Layers::Layer
+      NMath = Xumo::NMath
+
       attr_reader :alpha
 
       def self.load_hash(hash)
         self.new(hash[:alpha])
       end
@@ -120,20 +130,20 @@
         x1 = Xumo::SFloat.zeros(x.shape)
         x1[x >= 0] = 1
         x1 *= x
         x2 = Xumo::SFloat.zeros(x.shape)
         x2[x < 0] = 1
-        x2 *= @alpha * Xumo::NMath.exp(x) - @alpha
+        x2 *= @alpha * NMath.exp(x) - @alpha
         x1 + x2
       end
 
       def backward(dout)
         dx = Xumo::SFloat.ones(@x.shape)
         dx[@x < 0] = 0
         dx2 = Xumo::SFloat.zeros(@x.shape)
         dx2[@x < 0] = 1
-        dx2 *= @alpha * Xumo::NMath.exp(@x)
+        dx2 *= @alpha * NMath.exp(@x)
         dout * (dx + dx2)
       end
 
       def to_hash
         {class: self.class.name, alpha: @alpha}
@@ -150,11 +160,11 @@
         @out - y
       end
     
       def loss(y)
         batch_size = y.shape[0]
-        0.5 * ((@out - y)**2).sum / batch_size + ridge
+        0.5 * ((@out - y)**2).sum / batch_size + lasso + ridge
       end
     end
 
 
     class IdentityMAE < Layers::OutputLayer
@@ -169,11 +179,11 @@
         dout
       end
     
       def loss(y)
         batch_size = y.shape[0]
-        (@out - y).abs.sum / batch_size + ridge
+        (@out - y).abs.sum / batch_size + lasso + ridge
       end
     end
 
 
     class IdentityHuber < Layers::OutputLayer
@@ -181,11 +191,12 @@
         @out = x
       end
 
       def loss(y)
         loss = loss_l1(y)
-        @loss = loss > 1 ? loss : loss_l2(y)
+        loss = loss > 1 ? loss : loss_l2(y)
+        @loss = loss + lasso + ridge
       end
 
       def backward(y)
         dout = @out - y
         if @loss > 1
@@ -208,26 +219,30 @@
       end
     end
     
     
     class SoftmaxWithLoss < Layers::OutputLayer
+      NMath = Xumo::NMath
+
       def forward(x)
-        @out = Xumo::NMath.exp(x) / Xumo::NMath.exp(x).sum(1).reshape(x.shape[0], 1)
+        @out = NMath.exp(x) / NMath.exp(x).sum(1).reshape(x.shape[0], 1)
       end
     
       def backward(y)
         @out - y
       end
     
       def loss(y)
         batch_size = y.shape[0]
-        -(y * Xumo::NMath.log(@out + 1e-7)).sum / batch_size + ridge
+        -(y * NMath.log(@out + 1e-7)).sum / batch_size + lasso + ridge
       end
     end
 
 
     class SigmoidWithLoss < Layers::OutputLayer
+      NMath = Xumo::NMath
+
       def initialize
         @sigmoid = Sigmoid.new
       end
 
       def forward(x)
@@ -238,10 +253,10 @@
         @out - y
       end
 
       def loss(y)
         batch_size = y.shape[0]
-        -(y * Xumo::NMath.log(@out + 1e-7) + (1 - y) * Xumo::NMath.log(1 - @out + 1e-7)).sum / batch_size + ridge
+        -(y * NMath.log(@out + 1e-7) + (1 - y) * NMath.log(1 - @out + 1e-7)).sum / batch_size + lasso + ridge
       end
     end
 
   end
 end