lib/dnn/core/models.rb in ruby-dnn-0.13.4 vs lib/dnn/core/models.rb in ruby-dnn-0.14.0

- old
+ new

@@ -1,162 +1,173 @@ module DNN module Models - # This class deals with the model of the network. class Model attr_accessor :optimizer attr_accessor :loss_func + attr_reader :last_log # Load marshal model. # @param [String] file_name File name of marshal model to load. # @return [DNN::Models::Model] Return the loaded model. def self.load(file_name) - model = self.new + model = new loader = Loaders::MarshalLoader.new(model) loader.load(file_name) model end def initialize @optimizer = nil @loss_func = nil @last_link = nil @built = false - @callbacks = { - before_epoch: [], - after_epoch: [], - before_train_on_batch: [], - after_train_on_batch: [], - before_test_on_batch: [], - after_test_on_batch: [], - } + @callbacks = [] @layers_cache = nil + @last_log = {} end - # This method is provided for compatibility with v0.12.4. - # Load hash model parameters. - # @param [Hash] hash Hash to load model parameters. - def load_hash_params(hash) - has_param_layers_params = hash[:params] - has_param_layers_index = 0 - has_param_layers.uniq.each do |layer| - hash_params = has_param_layers_params[has_param_layers_index] - hash_params.each do |key, (shape, bin)| - data = Xumo::SFloat.from_binary(bin).reshape(*shape) - layer.get_params[key].data = data - end - has_param_layers_index += 1 - end - end - - # This method is provided for compatibility with v0.12.4. - # Load json model parameters. - # @param [String] json_str JSON string to load model parameters. - def load_json_params(json_str) - hash = JSON.parse(json_str, symbolize_names: true) - has_param_layers_params = hash[:params] - has_param_layers_index = 0 - has_param_layers.uniq.each do |layer| - hash_params = has_param_layers_params[has_param_layers_index] - hash_params.each do |key, (shape, base64_param)| - bin = Base64.decode64(base64_param) - data = Xumo::SFloat.from_binary(bin).reshape(*shape) - layer.get_params[key].data = data - end - has_param_layers_index += 1 - end - end - # Set optimizer and loss_func to model. # @param [DNN::Optimizers::Optimizer] optimizer Optimizer to use for learning. # @param [DNN::Losses::Loss] loss_func Loss function to use for learning. def setup(optimizer, loss_func) unless optimizer.is_a?(Optimizers::Optimizer) - raise TypeError.new("optimizer:#{optimizer.class} is not an instance of DNN::Optimizers::Optimizer class.") + raise TypeError, "optimizer:#{optimizer.class} is not an instance of DNN::Optimizers::Optimizer class." end unless loss_func.is_a?(Losses::Loss) - raise TypeError.new("loss_func:#{loss_func.class} is not an instance of DNN::Losses::Loss class.") + raise TypeError, "loss_func:#{loss_func.class} is not an instance of DNN::Losses::Loss class." end @optimizer = optimizer @loss_func = loss_func end # Start training. # Setup the model before use this method. # @param [Numo::SFloat] x Input training data. # @param [Numo::SFloat] y Output training data. # @param [Integer] epochs Number of training. - # @param [Integer] initial_epoch Initial epoch. # @param [Integer] batch_size Batch size used for one training. + # @param [Integer] initial_epoch Initial epoch. # @param [Array | NilClass] test If you to test the model for every 1 epoch, # specify [x_test, y_test]. Don't test to the model, specify nil. # @param [Boolean] verbose Set true to display the log. If false is set, the log is not displayed. def train(x, y, epochs, batch_size: 1, initial_epoch: 1, test: nil, verbose: true) - raise DNN_Error.new("The model is not optimizer setup complete.") unless @optimizer - raise DNN_Error.new("The model is not loss_func setup complete.") unless @loss_func check_xy_type(x, y) - iter = Iterator.new(x, y) - num_train_datas = x.is_a?(Array) ? x[0].shape[0] : x.shape[0] - (initial_epoch..epochs).each do |epoch| - call_callbacks(:before_epoch, epoch) - puts "【 epoch #{epoch}/#{epochs} 】" if verbose - iter.foreach(batch_size) do |x_batch, y_batch, index| - loss_value = train_on_batch(x_batch, y_batch) - if loss_value.is_a?(Xumo::SFloat) - loss_value = loss_value.mean - elsif loss_value.nan? - puts "\nloss is nan" if verbose - return - end - num_trained_datas = (index + 1) * batch_size - num_trained_datas = num_trained_datas > num_train_datas ? num_train_datas : num_trained_datas - log = "\r" - 40.times do |i| - if i < num_trained_datas * 40 / num_train_datas - log << "=" - elsif i == num_trained_datas * 40 / num_train_datas - log << ">" - else - log << "_" + train_iterator = Iterator.new(x, y) + train_by_iterator(train_iterator, epochs, + batch_size: batch_size, + initial_epoch: initial_epoch, + test: test, + verbose: verbose) + end + + alias fit train + + # Start training by iterator. + # Setup the model before use this method. + # @param [Iterator] train_iterator Iterator used for training. + # @param [Integer] epochs Number of training. + # @param [Integer] batch_size Batch size used for one training. + # @param [Integer] initial_epoch Initial epoch. + # @param [Array | NilClass] test If you to test the model for every 1 epoch, + # specify [x_test, y_test]. Don't test to the model, specify nil. + # @param [Boolean] verbose Set true to display the log. If false is set, the log is not displayed. + def train_by_iterator(train_iterator, epochs, + batch_size: 1, + initial_epoch: 1, + test: nil, + verbose: true) + raise DNN_Error, "The model is not optimizer setup complete." unless @optimizer + raise DNN_Error, "The model is not loss_func setup complete." unless @loss_func + + num_train_datas = train_iterator.num_datas + num_train_datas = num_train_datas / batch_size * batch_size if train_iterator.last_round_down + + stopped = catch(:stop) do + (initial_epoch..epochs).each do |epoch| + @last_log[:epoch] = epoch + call_callbacks(:before_epoch) + puts "【 epoch #{epoch}/#{epochs} 】" if verbose + + train_iterator.foreach(batch_size) do |x_batch, y_batch, index| + train_step_met = train_step(x_batch, y_batch) + num_trained_datas = (index + 1) * batch_size + num_trained_datas = num_trained_datas > num_train_datas ? num_train_datas : num_trained_datas + log = "\r" + 40.times do |i| + if i < num_trained_datas * 40 / num_train_datas + log << "=" + elsif i == num_trained_datas * 40 / num_train_datas + log << ">" + else + log << "_" + end end + + log << " #{num_trained_datas}/#{num_train_datas} " + log << metrics_to_str(train_step_met) + print log if verbose end - log << " #{num_trained_datas}/#{num_train_datas} loss: #{sprintf('%.8f', loss_value)}" - print log if verbose + + if test + test_met = test(test[0], test[1], batch_size: batch_size) + print " " + metrics_to_str(test_met) if verbose + end + puts "" if verbose + call_callbacks(:after_epoch) end - if test - acc, test_loss = accuracy(test[0], test[1], batch_size: batch_size) - print " accuracy: #{acc}, test loss: #{sprintf('%.8f', test_loss)}" if verbose - end - puts "" if verbose - call_callbacks(:after_epoch, epoch) + nil end + + if stopped + puts "\n#{stopped}" if verbose + end end - alias fit train + alias fit_by_iterator train_by_iterator + # Implement the training process to be performed in one step. + # @param [Numo::SFloat] x Input training data. + # @param [Numo::SFloat] y Output training data. + # @return [Hash] Hash of contents to be output to log. + private def train_step(x, y) + loss_value = train_on_batch(x, y) + { loss: loss_value.mean } + end + + # Implement the test process to be performed. + # @param [Numo::SFloat] x Input training data. + # @param [Numo::SFloat] y Output training data. + # @param [Integer] batch_size Batch size used for one test. + # @return [Hash] Hash of contents to be output to log. + private def test(x, y, batch_size: 100) + acc, test_loss = accuracy(x, y, batch_size: batch_size) + { accuracy: acc, test_loss: test_loss.mean } + end + # Training once. # Setup the model before use this method. # @param [Numo::SFloat] x Input training data. # @param [Numo::SFloat] y Output training data. - # @param [Integer] batch_size Batch size used for one test. # @return [Float | Numo::SFloat] Return loss value in the form of Float or Numo::SFloat. def train_on_batch(x, y) - raise DNN_Error.new("The model is not optimizer setup complete.") unless @optimizer - raise DNN_Error.new("The model is not loss_func setup complete.") unless @loss_func + raise DNN_Error, "The model is not optimizer setup complete." unless @optimizer + raise DNN_Error, "The model is not loss_func setup complete." unless @loss_func check_xy_type(x, y) call_callbacks(:before_train_on_batch) x = forward(x, true) loss_value = @loss_func.loss(x, y, layers) dy = @loss_func.backward(x, y) backward(dy) @optimizer.update(layers) @loss_func.regularizers_backward(layers) - call_callbacks(:after_train_on_batch, loss_value) + @last_log[:train_loss] = loss_value + call_callbacks(:after_train_on_batch) loss_value end # Evaluate model and get accuracy of test data. # @param [Numo::SFloat] x Input test data. @@ -167,19 +178,22 @@ check_xy_type(x, y) num_test_datas = x.is_a?(Array) ? x[0].shape[0] : x.shape[0] batch_size = batch_size >= num_test_datas[0] ? num_test_datas : batch_size iter = Iterator.new(x, y, random: false) total_correct = 0 - sum_loss = 0 + sum_loss = Xumo::SFloat[0] max_steps = (num_test_datas.to_f / batch_size).ceil iter.foreach(batch_size) do |x_batch, y_batch| correct, loss_value = test_on_batch(x_batch, y_batch) total_correct += correct - sum_loss += loss_value.is_a?(Xumo::SFloat) ? loss_value.mean : loss_value + sum_loss += loss_value.mean end mean_loss = sum_loss / max_steps - [total_correct.to_f / num_test_datas, mean_loss] + acc = total_correct.to_f / num_test_datas + @last_log[:test_loss] = mean_loss + @last_log[:test_accuracy] = acc + [acc, mean_loss] end # Evaluate once. # @param [Numo::SFloat] x Input test data. # @param [Numo::SFloat] y Output test data. @@ -187,11 +201,11 @@ def test_on_batch(x, y) call_callbacks(:before_test_on_batch) x = forward(x, false) correct = evaluate(x, y) loss_value = @loss_func.loss(x, y, layers) - call_callbacks(:after_test_on_batch, loss_value) + call_callbacks(:after_test_on_batch) [correct, loss_value] end # Implement the process to evaluate this model. # @param [Numo::SFloat] x Input test data. @@ -212,46 +226,37 @@ correct end # Predict data. # @param [Numo::SFloat] x Input data. - def predict(x) + # @param [Boolean] use_loss_activation Use loss activation when loss has an activation. + def predict(x, use_loss_activation: true) check_xy_type(x) - forward(x, false) + y = forward(x, false) + if use_loss_activation && @loss_func.class.respond_to?(:activation) + y = @loss_func.class.activation(y) + end + y end # Predict one data. # @param [Numo::SFloat] x Input data. However, x is single data. - def predict1(x) + def predict1(x, use_loss_activation: true) check_xy_type(x) - predict(x.reshape(1, *x.shape))[0, false] + predict(x.reshape(1, *x.shape), use_loss_activation: use_loss_activation)[0, false] end # Add callback function. - # @param [Symbol] event Callback event. The following can be used for event. - # before_epoch: Process: performed before one training. - # after_epoch: Process: performed after one training. - # before_train_on_batch: Set the proc to be performed before train on batch processing. - # after_train_on_batch: Set the proc to be performed after train on batch processing. - # before_test_on_batch: Set the proc to be performed before test on batch processing. - # after_test_on_batch: Set the proc to be performed after test on batch processing. - def add_callback(event, callback) - raise DNN_UnknownEventError.new("Unknown event #{event}.") unless @callbacks.has_key?(event) - @callbacks[event] << callback + # @param [Callback] callback Callback object. + def add_callback(callback) + callback.model = self + @callbacks << callback end # Clear the callback function registered for each event. - # @param [Symbol] event Callback event. The following can be used for event. - # before_epoch: Process: performed before one training. - # after_epoch: Process: performed after one training. - # before_train_on_batch: Set the proc to be performed before train on batch processing. - # after_train_on_batch: Set the proc to be performed after train on batch processing. - # before_test_on_batch: Set the proc to be performed before test on batch processing. - # after_test_on_batch: Set the proc to be performed after test on batch processing. - def clear_callbacks(event) - raise DNN_UnknownEventError.new("Unknown event #{event}.") unless @callbacks.has_key?(event) - @callbacks[event] = [] + def clear_callbacks + @callbacks = [] end # Save the model in marshal format. # @param [String] file_name Name to save model. def save(file_name) @@ -265,11 +270,11 @@ end # Get the all layers. # @return [Array] All layers array. def layers - raise DNN_Error.new("This model is not built. You need build this model using predict or train.") unless built? + raise DNN_Error, "This model is not built. You need build this model using predict or train." unless built? return @layers_cache if @layers_cache layers = [] get_layers = -> link do return unless link layers.unshift(link.layer) @@ -289,11 +294,11 @@ def has_param_layers layers.select { |layer| layer.is_a?(Layers::HasParamLayer) } end # Get the layer that the model has. - # @param [Symbol] The name of the layer to get. + # @param [Symbol] name The name of the layer to get. # @return [DNN::Layers::Layer] Return the layer. def get_layer(name) layers.find { |layer| layer.name == name } end @@ -305,25 +310,26 @@ private def forward(x, learning_phase) DNN.learning_phase = learning_phase @layers_cache = nil - y, @last_link = call(x) + output_tensor = call(Tensor.new(x, nil)) + @last_link = output_tensor.link unless @built @built = true naming end - y + output_tensor.data end def backward(dy) @last_link.backward(dy) end - def call_callbacks(event, *args) - @callbacks[event].each do |callback| - callback.call(*args) + def call_callbacks(event) + @callbacks.each do |callback| + callback.send(event) if callback.respond_to?(event) end end def naming layers.each do |layer| @@ -336,21 +342,24 @@ end end end end + def metrics_to_str(mertics) + mertics.map { |key, num| "#{key}: #{sprintf('%.4f', num)}" }.join(", ") + end + def check_xy_type(x, y = nil) if !x.is_a?(Xumo::SFloat) && !x.is_a?(Array) - raise TypeError.new("x:#{x.class.name} is not an instance of #{Xumo::SFloat.name} class or Array class.") + raise TypeError, "x:#{x.class.name} is not an instance of #{Xumo::SFloat.name} class or Array class." end if y && !y.is_a?(Xumo::SFloat) && !x.is_a?(Array) - raise TypeError.new("y:#{y.class.name} is not an instance of #{Xumo::SFloat.name} class or Array class.") + raise TypeError, "y:#{y.class.name} is not an instance of #{Xumo::SFloat.name} class or Array class." end end end - class Sequential < Model attr_reader :stack # @param [Array] stack All layers possessed by the model. def initialize(stack = []) @@ -361,11 +370,11 @@ # Add layer to the model. # @param [DNN::Layers::Layer] layer Layer to add to the model. # @return [DNN::Models::Model] Return self. def add(layer) unless layer.is_a?(Layers::Layer) || layer.is_a?(Model) - raise TypeError.new("layer: #{layer.class.name} is not an instance of the DNN::Layers::Layer class or DNN::Models::Model class.") + raise TypeError, "layer: #{layer.class.name} is not an instance of the DNN::Layers::Layer class or DNN::Models::Model class." end @stack << layer self end @@ -374,10 +383,10 @@ # Remove layer to the model. # @param [DNN::Layers::Layer] layer Layer to remove to the model. # @return [Boolean] Return true if success for remove layer. def remove(layer) unless layer.is_a?(Layers::Layer) || layer.is_a?(Model) - raise TypeError.new("layer: #{layer.class.name} is not an instance of the DNN::Layers::Layer class or DNN::Models::Model class.") + raise TypeError, "layer: #{layer.class.name} is not an instance of the DNN::Layers::Layer class or DNN::Models::Model class." end @stack.delete(layer) ? true : false end def call(x)