module DNN module Layers # This module is used for convolution. module Conv2DModule private # img[bsize, out_h, out_w, channel] to col[bsize * out_h * out_w, fil_h * fil_w * ch] def im2col(img, out_h, out_w, fil_h, fil_w, strides) bsize = img.shape[0] ch = img.shape[3] col = Xumo::SFloat.zeros(bsize, out_h, out_w, fil_h, fil_w, ch) (0...fil_h).each do |i| i_range = (i...(i + strides[0] * out_h)).step(strides[0]).to_a (0...fil_w).each do |j| j_range = (j...(j + strides[1] * out_w)).step(strides[1]).to_a col[true, true, true, i, j, true] = img[true, i_range, j_range, true] end end col.reshape(bsize * out_h * out_w, fil_h * fil_w * ch) end # col[bsize * out_h * out_w, fil_h * fil_w * ch] to img[bsize, out_h, out_w, channel] def col2im(col, img_shape, out_h, out_w, fil_h, fil_w, strides) bsize, img_h, img_w, ch = img_shape col = col.reshape(bsize, out_h, out_w, fil_h, fil_w, ch) img = Xumo::SFloat.zeros(bsize, img_h, img_w, ch) (0...fil_h).each do |i| i_range = (i...(i + strides[0] * out_h)).step(strides[0]).to_a (0...fil_w).each do |j| j_range = (j...(j + strides[1] * out_w)).step(strides[1]).to_a img[true, i_range, j_range, true] += col[true, true, true, i, j, true] end end img end def padding(img, pad) bsize, img_h, img_w, ch = img.shape img2 = Xumo::SFloat.zeros(bsize, img_h + pad[0], img_w + pad[1], ch) i_begin = pad[0] / 2 i_end = i_begin + img_h j_begin = pad[1] / 2 j_end = j_begin + img_w img2[true, i_begin...i_end, j_begin...j_end, true] = img img2 end def back_padding(img, pad) i_begin = pad[0] / 2 i_end = img.shape[1] - (pad[0] / 2.0).round j_begin = pad[1] / 2 j_end = img.shape[2] - (pad[1] / 2.0).round img[true, i_begin...i_end, j_begin...j_end, true] end def out_size(prev_h, prev_w, fil_h, fil_w, strides) out_h = (prev_h - fil_h) / strides[0] + 1 out_w = (prev_w - fil_w) / strides[1] + 1 [out_h, out_w] end def padding_size(prev_h, prev_w, out_h, out_w, strides) pad_h = (prev_h.to_f / strides[0]).ceil - out_h pad_w = (prev_w.to_f / strides[1]).ceil - out_w [pad_h, pad_w] end end class Conv2D < Connection include Conv2DModule # @return [Integer] number of filters. attr_reader :num_filters # @return [Array] Return filter size. filter size is of the form [height, width]. attr_reader :filter_size # @return [Array] Return stride length. stride length is of the form [height, width]. attr_reader :strides def self.load_hash(hash) Conv2D.new(hash[:num_filters], hash[:filter_size], weight_initializer: Utils.load_hash(hash[:weight_initializer]), bias_initializer: Utils.load_hash(hash[:bias_initializer]), strides: hash[:strides], padding: hash[:padding], l1_lambda: hash[:l1_lambda], l2_lambda: hash[:l2_lambda]) end # @param [Integer] num_filters number of filters. # @param [Array or Integer] filter_size filter size. filter size is of the form [height, width]. # @param [Array or Integer] strides stride length. stride length is of the form [height, width]. # @param [Bool] padding Whether to padding. def initialize(num_filters, filter_size, weight_initializer: Initializers::RandomNormal.new, bias_initializer: Initializers::RandomNormal.new, strides: 1, padding: false, l1_lambda: 0, l2_lambda: 0) super(weight_initializer: weight_initializer, bias_initializer: bias_initializer, l1_lambda: l1_lambda, l2_lambda: l2_lambda) @num_filters = num_filters @filter_size = filter_size.is_a?(Integer) ? [filter_size, filter_size] : filter_size @strides = strides.is_a?(Integer) ? [strides, strides] : strides @padding = padding end def build(input_shape) super prev_h, prev_w = input_shape[0..1] @out_size = out_size(prev_h, prev_w, *@filter_size, @strides) if @padding @pad_size = padding_size(prev_h, prev_w, *@out_size, @strides) @out_size = [@out_size[0] + @pad_size[0], @out_size[1] + @pad_size[1]] end end def forward(x) x = padding(x, @pad_size) if @padding @x_shape = x.shape @col = im2col(x, *@out_size, *@filter_size, @strides) out = @col.dot(@weight.data) + @bias.data out.reshape(x.shape[0], *@out_size, out.shape[3]) end def backward(dout) dout = dout.reshape(dout.shape[0..2].reduce(:*), dout.shape[3]) @weight.grad = @col.transpose.dot(dout) @bias.grad = dout.sum(0) dcol = dout.dot(@weight.data.transpose) dx = col2im(dcol, @x_shape, *@out_size, *@filter_size, @strides) @padding ? back_padding(dx, @pad_size) : dx end def output_shape [*@out_size, @num_filters] end # @return [Bool] whether to padding. def padding? @padding end # @return [Numo::SFloat] Convert weight to filter and return. def filters num_prev_filter = @input_shape[2] @weight.data.reshape(*@filter_size, num_prev_filter, @num_filters) end # @param [Numo::SFloat] filters Convert weight to filters and set. def filters=(filters) num_prev_filter = @input_shape[2] @weight.data = filters.reshape(@filter_size.reduce(:*) * num_prev_filter, @num_filters) end def to_hash super({num_filters: @num_filters, filter_size: @filter_size, strides: @strides, padding: @padding}) end private def init_params num_prev_filter = @input_shape[2] @weight.data = Xumo::SFloat.new(@filter_size.reduce(:*) * num_prev_filter, @num_filters) @bias.data = Xumo::SFloat.new(@num_filters) super() end end # Super class of all pooling2D class. class Pool2D < Layer include Conv2DModule # @return [Array] Return pooling size. pooling size is of the form [height, width]. attr_reader :pool_size # @return [Array] Return stride length. stride length is of the form [height, width]. attr_reader :strides def self.load_hash(pool2d_class, hash) pool2d_class.new(hash[:pool_size], strides: hash[:strides], padding: hash[:padding]) end # @param [Array or Integer] pool_size pooling size. pooling size is of the form [height, width]. # @param [Array or Integer or NilClass] strides stride length. stride length is of the form [height, width]. # If you set nil, treat pool_size as strides. # @param [Bool] padding Whether to padding. def initialize(pool_size, strides: nil, padding: false) super() @pool_size = pool_size.is_a?(Integer) ? [pool_size, pool_size] : pool_size @strides = if strides strides.is_a?(Integer) ? [strides, strides] : strides else @pool_size.clone end @padding = padding end def build(input_shape) super prev_h, prev_w = input_shape[0..1] @num_channel = input_shape[2] @out_size = out_size(prev_h, prev_w, *@pool_size, @strides) if @padding @pad_size = padding_size(prev_h, prev_w, *@out_size, @strides) @out_size = [@out_size[0] + @pad_size[0], @out_size[1] + @pad_size[1]] end end def output_shape [*@out_size, @num_channel] end # @return [Bool] whether to padding. def padding? @padding end def to_hash super({pool_size: @pool_size, strides: @strides, padding: @padding}) end end class MaxPool2D < Pool2D def self.load_hash(hash) Pool2D.load_hash(self, hash) end def forward(x) x = padding(x, @pad_size) if @padding @x_shape = x.shape col = im2col(x, *@out_size, *@pool_size, @strides) col = col.reshape(x.shape[0] * @out_size.reduce(:*), @pool_size.reduce(:*), x.shape[3]).transpose(0, 2, 1) .reshape(x.shape[0] * @out_size.reduce(:*) * x.shape[3], @pool_size.reduce(:*)) @max_index = col.max_index(1) col.max(1).reshape(x.shape[0], *@out_size, x.shape[3]) end def backward(dout) dmax = Xumo::SFloat.zeros(dout.size * @pool_size.reduce(:*)) dmax[@max_index] = dout.flatten dcol = dmax.reshape(dout.shape[0..2].reduce(:*), @pool_size.reduce(:*) * dout.shape[3]) dx = col2im(dcol, @x_shape, *@out_size, *@pool_size, @strides) @padding ? back_padding(dx, @pad_size) : dx end end class AvgPool2D < Pool2D def self.load_hash(hash) Pool2D.load_hash(self, hash) end def forward(x) x = padding(x, @pad_size) if @padding @x_shape = x.shape col = im2col(x, *@out_size, *@pool_size, @strides) col = col.reshape(x.shape[0] * @out_size.reduce(:*), @pool_size.reduce(:*), x.shape[3]).transpose(0, 2, 1) .reshape(x.shape[0] * @out_size.reduce(:*) * x.shape[3], @pool_size.reduce(:*)) col.mean(1).reshape(x.shape[0], *@out_size, x.shape[3]) end def backward(dout) row_length = @pool_size.reduce(:*) dout /= row_length davg = Xumo::SFloat.zeros(dout.size, row_length) row_length.times do |i| davg[true, i] = dout.flatten end dcol = davg.reshape(dout.shape[0..2].reduce(:*), dout.shape[3] * @pool_size.reduce(:*)) dx = col2im(dcol, @x_shape, *@out_size, *@pool_size, @strides) @padding ? back_padding(dx, @pad_size) : dx end end class UnPool2D < Layer # @return [Array] Return unpooling size. unpooling size is of the form [height, width]. attr_reader :unpool_size # @param [Array or Integer] unpool_size Unpooling size. unpooling size is of the form [height, width]. def initialize(unpool_size) super() @unpool_size = unpool_size.is_a?(Integer) ? [unpool_size, unpool_size] : unpool_size end def self.load_hash(hash) UnPool2D.new(hash[:unpool_size]) end def build(input_shape) super prev_h, prev_w = input_shape[0..1] unpool_h, unpool_w = @unpool_size out_h = prev_h * unpool_h out_w = prev_w * unpool_w @out_size = [out_h, out_w] @num_channel = input_shape[2] end def forward(x) @x_shape = x.shape unpool_h, unpool_w = @unpool_size x2 = Xumo::SFloat.zeros(x.shape[0], x.shape[1], unpool_h, x.shape[2], unpool_w, @num_channel) x2[true, true, 0, true, 0, true] = x x2.reshape(x.shape[0], *@out_size, x.shape[3]) end def backward(dout) unpool_h, unpool_w = @unpool_size dout = dout.reshape(dout.shape[0], @x_shape[1], unpool_h, @x_shape[2], unpool_w, @num_channel) dout[true, true, 0, true, 0, true].clone end def output_shape [*@out_size, @num_channel] end def to_hash super({unpool_size: @unpool_size}) end end end end