lasso.rb in svmkit-0.6.1

- old
+ new

@@ -1,11 +1,10 @@
 # frozen_string_literal: true
 
 require 'svmkit/validation'
-require 'svmkit/base/base_estimator'
+require 'svmkit/linear_model/sgd_linear_estimator'
 require 'svmkit/base/regressor'
-require 'svmkit/optimizer/nadam'
 
 module SVMKit
   module LinearModel
     # Lasso is a class that implements Lasso Regression
     # with stochastic gradient descent (SGD) optimization.
@@ -17,12 +16,11 @@
     #   results = estimator.predict(testing_samples)
     #
     # *Reference*
     # - S. Shalev-Shwartz and Y. Singer, "Pegasos: Primal Estimated sub-GrAdient SOlver for SVM," Proc. ICML'07, pp. 807--814, 2007.
     # - L. Bottou, "Large-Scale Machine Learning with Stochastic Gradient Descent," Proc. COMPSTAT'10, pp. 177--186, 2010.
-    class Lasso
-      include Base::BaseEstimator
+    class Lasso < SGDLinearEstimator
       include Base::Regressor
       include Validation
 
       # Return the weight vector.
       # @return [Numo::DFloat] (shape: [n_outputs, n_features])
@@ -38,33 +36,23 @@
 
       # Create a new Lasso regressor.
       #
       # @param reg_param [Float] The regularization parameter.
       # @param fit_bias [Boolean] The flag indicating whether to fit the bias term.
+      # @param bias_scale [Float] The scale of the bias term.
       # @param max_iter [Integer] The maximum number of iterations.
       # @param batch_size [Integer] The size of the mini batches.
       # @param optimizer [Optimizer] The optimizer to calculate adaptive learning rate.
       #   If nil is given, Nadam is used.
       # @param random_seed [Integer] The seed value using to initialize the random generator.
-      def initialize(reg_param: 1.0, fit_bias: false, max_iter: 1000, batch_size: 10, optimizer: nil, random_seed: nil)
-        check_params_float(reg_param: reg_param)
+      def initialize(reg_param: 1.0, fit_bias: false, bias_scale: 1.0, max_iter: 1000, batch_size: 10, optimizer: nil, random_seed: nil)
+        check_params_float(reg_param: reg_param, bias_scale: bias_scale)
         check_params_integer(max_iter: max_iter, batch_size: batch_size)
         check_params_boolean(fit_bias: fit_bias)
         check_params_type_or_nil(Integer, random_seed: random_seed)
         check_params_positive(reg_param: reg_param, max_iter: max_iter, batch_size: batch_size)
-        @params = {}
-        @params[:reg_param] = reg_param
-        @params[:fit_bias] = fit_bias
-        @params[:max_iter] = max_iter
-        @params[:batch_size] = batch_size
-        @params[:optimizer] = optimizer
-        @params[:optimizer] ||= Optimizer::Nadam.new
-        @params[:random_seed] = random_seed
-        @params[:random_seed] ||= srand
-        @weight_vec = nil
-        @bias_term = nil
-        @rng = Random.new(@params[:random_seed])
+        super
       end
 
       # Fit the model with given training data.
       #
       # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
@@ -74,18 +62,18 @@
         check_sample_array(x)
         check_tvalue_array(y)
         check_sample_tvalue_size(x, y)
 
         n_outputs = y.shape[1].nil? ? 1 : y.shape[1]
-        _n_samples, n_features = x.shape
+        n_features = x.shape[1]
 
         if n_outputs > 1
           @weight_vec = Numo::DFloat.zeros(n_outputs, n_features)
           @bias_term = Numo::DFloat.zeros(n_outputs)
-          n_outputs.times { |n| @weight_vec[n, true], @bias_term[n] = single_fit(x, y[true, n]) }
+          n_outputs.times { |n| @weight_vec[n, true], @bias_term[n] = partial_fit(x, y[true, n]) }
         else
-          @weight_vec, @bias_term = single_fit(x, y)
+          @weight_vec, @bias_term = partial_fit(x, y)
         end
 
         self
       end
 
@@ -117,62 +105,34 @@
         nil
       end
 
       private
 
-      def single_fit(x, y)
-        # Expand feature vectors for bias term.
-        samples = @params[:fit_bias] ? expand_feature(x) : x
-        # Initialize some variables.
-        n_samples, n_features = samples.shape
-        rand_ids = [*0...n_samples].shuffle(random: @rng)
-        weight_vec = Numo::DFloat.zeros(n_features)
-        left_weight_vec = Numo::DFloat.zeros(n_features)
-        right_weight_vec = Numo::DFloat.zeros(n_features)
-        left_optimizer = @params[:optimizer].dup
-        right_optimizer = @params[:optimizer].dup
-        # Start optimization.
-        @params[:max_iter].times do |_t|
-          # Random sampling.
-          subset_ids = rand_ids.shift(@params[:batch_size])
-          rand_ids.concat(subset_ids)
-          data = samples[subset_ids, true]
-          values = y[subset_ids]
-          # Calculate gradients for loss function.
-          loss_grad = loss_gradient(data, values, weight_vec)
-          next if loss_grad.ne(0.0).count.zero?
-          # Update weight.
-          left_weight_vec = round_weight(left_optimizer.call(left_weight_vec, left_weight_gradient(loss_grad, data)))
-          right_weight_vec = round_weight(right_optimizer.call(right_weight_vec, right_weight_gradient(loss_grad, data)))
-          weight_vec = left_weight_vec - right_weight_vec
-        end
-        split_weight_vec_bias(weight_vec)
+      def partial_fit(x, y)
+        n_features = @params[:fit_bias] ? x.shape[1] + 1 : x.shape[1]
+        @left_weight = Numo::DFloat.zeros(n_features)
+        @right_weight = Numo::DFloat.zeros(n_features)
+        @left_optimizer = @params[:optimizer].dup
+        @right_optimizer = @params[:optimizer].dup
+        super
       end
 
-      def loss_gradient(x, y, weight)
+      def calc_loss_gradient(x, y, weight)
         2.0 * (x.dot(weight) - y)
       end
 
-      def left_weight_gradient(loss_grad, data)
-        ((@params[:reg_param] + loss_grad).expand_dims(1) * data).mean(0)
+      def calc_new_weight(_optimizer, x, _weight, loss_gradient)
+        @left_weight = round_weight(@left_optimizer.call(@left_weight, calc_weight_gradient(loss_gradient, x)))
+        @right_weight = round_weight(@right_optimizer.call(@right_weight, calc_weight_gradient(-loss_gradient, x)))
+        @left_weight - @right_weight
       end
 
-      def right_weight_gradient(loss_grad, data)
-        ((@params[:reg_param] - loss_grad).expand_dims(1) * data).mean(0)
+      def calc_weight_gradient(loss_gradient, data)
+        ((@params[:reg_param] + loss_gradient).expand_dims(1) * data).mean(0)
       end
 
       def round_weight(weight)
         0.5 * (weight + weight.abs)
-      end
-
-      def expand_feature(x)
-        Numo::NArray.hstack([x, Numo::DFloat.ones([x.shape[0], 1])])
-      end
-
-      def split_weight_vec_bias(weight_vec)
-        weights = @params[:fit_bias] ? weight_vec[0...-1] : weight_vec
-        bias = @params[:fit_bias] ? weight_vec[-1] : 0.0
-        [weights, bias]
       end
     end
   end
 end