lib/torch/nn/functional.rb in torch-rb-0.1.5 vs lib/torch/nn/functional.rb in torch-rb-0.1.6
- old
+ new
@@ -1,89 +1,373 @@
module Torch
module NN
class Functional
class << self
+ include Utils
+
+ # convolution layers
+
+ def conv1d(*args, **options)
+ Torch.conv1d(*args, **options)
+ end
+
+ def conv2d(*args, **options)
+ Torch.conv2d(*args, **options)
+ end
+
+ def conv3d(*args, **options)
+ Torch.conv3d(*args, **options)
+ end
+
+ def unfold(input, kernel_size, dilation: 1, padding: 0, stride: 1)
+ if input.dim == 4
+ NN.im2col(input, _pair(kernel_size), _pair(dilation), _pair(padding), _pair(stride))
+ else
+ raise Error, "Input Error: Only 4D input Tensors are supported (got #{input.dim}D)"
+ end
+ end
+
+ def fold(input, output_size, kernel_size, dilation: 1, padding: 0, stride: 1)
+ if input.dim == 3
+ NN.col2im(input, _pair(output_size), _pair(kernel_size), _pair(dilation), _pair(padding), _pair(stride))
+ else
+ raise Error, "Input Error: Only 3D input Tensors are supported (got #{input.dim}D)"
+ end
+ end
+
+ # pooling layers
+
+ def max_pool1d(*args, **options)
+ return_indices = args.pop if args.size == 7
+ if return_indices
+ Torch.max_pool1d_with_indices(*args, **options)
+ else
+ Torch.max_pool1d(*args, **options)
+ end
+ end
+
+ def max_pool2d(*args, **options)
+ return_indices = args.pop if args.size == 7
+ if return_indices
+ NN.max_pool2d_with_indices(*args, **options)
+ else
+ Torch.max_pool2d(*args, **options)
+ end
+ end
+
+ def max_pool3d(*args, **options)
+ return_indices = args.pop if args.size == 7
+ if return_indices
+ NN.max_pool3d_with_indices(*args, **options)
+ else
+ Torch.max_pool3d(*args, **options)
+ end
+ end
+
+ def max_unpool1d(input, indices, kernel_size, stride: nil, padding: 0, output_size: nil)
+ raise NotImplementedYet
+ kernel_size = _single(kernel_size)
+ if !stride.nil?
+ _stride = _single(stride)
+ else
+ _stride = kernel_size
+ end
+ padding = _single(padding)
+ output_size = _unpool_output_size(input, kernel_size, _stride, padding, output_size)
+ output_size = output_size + [1]
+ NN.max_unpool2d(input.unsqueeze(3), indices.unsqueeze(3), output_size).squeeze(3)
+ end
+
+ def max_unpool2d(*args, **options)
+ raise NotImplementedYet
+ NN.max_unpool2d(*args, **options)
+ end
+
+ def max_unpool3d(*args, **options)
+ raise NotImplementedYet
+ NN.max_unpool3d(*args, **options)
+ end
+
+ def avg_pool1d(*args, **options)
+ Torch.avg_pool1d(*args, **options)
+ end
+
+ def avg_pool2d(*args, **options)
+ NN.avg_pool2d(*args, **options)
+ end
+
+ def avg_pool3d(*args, **options)
+ NN.avg_pool3d(*args, **options)
+ end
+
+ # padding layers
+
+ def pad(input, pad, mode: "constant", value: 0)
+ raise ArgumentError, "Padding length must be divisible by 2" unless pad.size % 2 == 0
+ raise ArgumentError, "Padding length too large" unless pad.size / 2 <= input.dim
+
+ if mode == "constant"
+ return Torch.constant_pad_nd(input, pad, value)
+ else
+ raise ArgumentError, "Padding mode doesn't take in value argument" unless value == 0
+
+ if input.dim == 3
+ raise ArgumentError, "3D tensors expect 2 values for padding" unless pad.size == 2
+ case mode
+ when "reflect"
+ NN.reflection_pad1d(input, pad)
+ when "replicate"
+ NN.replication_pad1d(input, pad)
+ else
+ raise NotImplementedYet
+ end
+ elsif input.dim == 4
+ raise ArgumentError, "4D tensors expect 4 values for padding" unless pad.size == 4
+ case mode
+ when "reflect"
+ NN.reflection_pad2d(input, pad)
+ when "replicate"
+ NN.replication_pad2d(input, pad)
+ else
+ raise NotImplementedYet
+ end
+ elsif input.dim == 5
+ raise ArgumentError, "5D tensors expect 6 values for padding" unless pad.size == 6
+ case mode
+ when "replicate"
+ NN.replication_pad3d(input, pad)
+ else
+ raise NotImplementedYet
+ end
+ else
+ raise ArgumentError, "Only 3D, 4D, 5D padding with non-constant padding are supported for now"
+ end
+ end
+ end
+
+ # activation layers
+
+ def hardshrink(input, lambd = 0.5)
+ Torch.hardshrink(input, lambd)
+ end
+
+ def leaky_relu(input, negative_slope = 0.01)
+ NN.leaky_relu(input, negative_slope)
+ end
+
+ def log_sigmoid(input)
+ NN.log_sigmoid(input)
+ end
+
+ def prelu(input, weight)
+ Torch.prelu(input, weight)
+ end
+
def relu(input, inplace: false)
if inplace
input.relu!
else
input.relu
end
end
- def conv2d(input, weight, bias, stride: 1, padding: 0, dilation: 1, groups: 1)
- # TODO pair stride and padding when needed
- Torch.conv2d(input, weight, bias, stride, padding, dilation, groups)
+ def softplus(input, beta: 1, threshold: 20)
+ NN.softplus(input, beta, threshold)
end
- def prelu(input, weight)
- Torch.prelu(input, weight)
+ def softshrink(*args, **options)
+ NN.softshrink(*args, **options)
end
- def leaky_relu(input, negative_slope = 0.01)
- Torch.leaky_relu(input, negative_slope)
+ def softsign(input)
+ input / (input.abs + 1)
end
- def max_pool2d(input, kernel_size)
- kernel_size = [kernel_size, kernel_size] if kernel_size.is_a?(Integer)
- Torch.max_pool2d(input, kernel_size)
+ def tanhshrink(input)
+ input - input.tanh
end
- def avg_pool2d(input, kernel_size)
- kernel_size = [kernel_size, kernel_size] if kernel_size.is_a?(Integer)
- Torch.avg_pool2d(input, kernel_size)
+ # other activation layers
+
+ def softmin(input, dim: nil)
+ dim ||= softmax_dim(input.dim)
+ (-input).softmax(dim)
end
+ def softmax(input, dim: nil)
+ dim ||= softmax_dim(input.dim)
+ input.softmax(dim)
+ end
+
+ # TODO make dim keyword argument and update examples
+ def log_softmax(input, dim = nil)
+ dim ||= softmax_dim(input.dim)
+ input.log_softmax(dim)
+ end
+
+ # normalization layers
+
+ def batch_norm(input, running_mean, running_var, weight: nil, bias: nil,
+ training: false, momentum: 0.1, eps: 1e-5)
+
+ if training
+ size = input.size
+ size_prods = size[0]
+ (size.length - 2).times do |i|
+ size_prods *= size[i + 2]
+ end
+ if size_prods == 1
+ raise ArgumentError, "Expected more than 1 value per channel when training, got input size #{size.inspect}"
+ end
+ end
+
+ Torch.batch_norm(
+ input, weight, bias, running_mean, running_var,
+ training, momentum, eps, false
+ )
+ end
+
+ def group_norm(input, num_groups, weight: nil, bias: nil, eps: 1e-5)
+ Torch.group_norm(input, num_groups, weight, bias, eps, false)
+ end
+
+ def instance_norm(input, running_mean: nil, running_var: nil, weight: nil,
+ bias: nil, use_input_stats: true, momentum: 0.1, eps: 1e-5)
+
+ Torch.instance_norm(
+ input, weight, bias, running_mean, running_var,
+ use_input_stats, momentum, eps, false
+ )
+ end
+
+ def layer_norm(input, normalized_shape, weight: nil, bias: nil, eps: 1e-5)
+ Torch.layer_norm(input, normalized_shape, weight, bias, eps, false)
+ end
+
+ def local_response_norm(input, size, alpha: 1e-4, beta: 0.75, k: 1.0)
+ dim = input.dim
+ if dim < 3
+ raise ArgumentError, "Expected 3D or higher dimensionality input (got #{dim} dimensions)"
+ end
+ div = input.mul(input).unsqueeze(1)
+ if dim == 3
+ div = pad(div, [0, 0, size / 2, (size - 1) / 2])
+ div = avg_pool2d(div, [size, 1], stride: 1).squeeze(1)
+ else
+ sizes = input.size
+ div = div.view(sizes[0], 1, sizes[1], sizes[2], -1)
+ div = pad(div, [0, 0, 0, 0, size / 2, (size - 1) / 2])
+ div = avg_pool3d(div, [size, 1, 1], stride: 1).squeeze(1)
+ div = div.view(sizes)
+ end
+ div = div.mul(alpha).add(k).pow(beta)
+ input / div
+ end
+
# linear layers
+ def linear(input, weight, bias)
+ NN.linear(input, weight, bias)
+ end
+
def bilinear(input1, input2, weight, bias)
Torch.bilinear(input1, input2, weight, bias)
end
- def linear(input, weight, bias)
- Torch.linear(input, weight, bias)
+ # dropout layers
+
+ def dropout(input, p: 0.5, training: true, inplace: false)
+ if inplace
+ Torch.dropout!(input, p, training)
+ else
+ Torch.dropout(input, p, training)
+ end
end
+ def dropout2d(input, p: 0.5, training: true, inplace: false)
+ raise ArgumentError, "dropout probability has to be between 0 and 1, but got #{p}" if p < 0 || p > 1
+
+ if inplace
+ Torch.feature_dropout!(input, p, training)
+ else
+ Torch.feature_dropout(input, p, training)
+ end
+ end
+
+ def dropout3d(input, p: 0.5, training: true, inplace: false)
+ if inplace
+ Torch.feature_dropout!(input, p, training)
+ else
+ Torch.feature_dropout(input, p, training)
+ end
+ end
+
+ def alpha_dropout(input, p: 0.5, training: true, inplace: false)
+ if inplace
+ Torch.alpha_dropout!(input, p, training)
+ else
+ Torch.alpha_dropout(input, p, training)
+ end
+ end
+
+ def feature_alpha_dropout(input, p: 0.5, training: true, inplace: false)
+ if inplace
+ Torch.feature_alpha_dropout!(input, p, training)
+ else
+ Torch.feature_alpha_dropout(input, p, training)
+ end
+ end
+
# sparse layers
def embedding(input, weight, padding_idx: nil, max_norm: nil, norm_type: 2.0, scale_grad_by_freq: false, sparse: false)
# TODO handle max_norm and norm_type
raise NotImplementedYet unless max_norm.nil? && norm_type == 2.0
padding_idx ||= -1
# weight and indices are swapped from Python interface
- Torch._embedding(weight, input, padding_idx, scale_grad_by_freq, sparse)
+ Torch.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse)
end
def embedding_bag(input, weight, offsets: nil, max_norm: nil, norm_type: 2, scale_grad_by_freq: false, mode: "mean", sparse: false, per_sample_weights: nil)
- # need to handle nils
- raise NotImplementedYet
-
# TODO handle max_norm and norm_type
raise NotImplementedYet unless max_norm.nil? && norm_type == 2.0
- Torch._embedding_bag(input, weight, offsets, scale_grad_by_freq, mode, sparse, per_sample_weights)
+ mode_enum =
+ case mode
+ when "sum"
+ 0
+ when "mean"
+ 1
+ when "max"
+ 2
+ else
+ raise ArgumentError, "Unknown mode: #{mode}"
+ end
+
+ # weight and input swapped
+ Torch.embedding_bag(weight, input, offsets, scale_grad_by_freq, mode_enum, sparse, per_sample_weights)
end
# distance functions
def cosine_similarity(x1, x2, dim: 1, eps: 1e-8)
- Torch._cosine_similarity(x1, x2, dim, eps)
+ Torch.cosine_similarity(x1, x2, dim, eps)
end
def pairwise_distance(x1, x2, p: 2.0, eps: 1e-6, keepdim: false)
- Torch._pairwise_distance(x1, x2, p, eps, keepdim)
+ Torch.pairwise_distance(x1, x2, p, eps, keepdim)
end
# loss functions
def binary_cross_entropy(input, target, weight: nil, reduction: "mean")
- NN._binary_cross_entropy(input, target, weight, reduction)
+ NN.binary_cross_entropy(input, target, weight, reduction)
end
def binary_cross_entropy_with_logits(input, target, weight: nil, reduction: "mean", pos_weight: nil)
- Torch._binary_cross_entropy_with_logits(input, target, weight, pos_weight, reduction)
+ Torch.binary_cross_entropy_with_logits(input, target, weight, pos_weight, reduction)
end
def cosine_embedding_loss(input1, input2, target, margin: 0, reduction: "mean")
raise NotImplementedYet
end
@@ -92,126 +376,62 @@
nll_loss(log_softmax(input, 1), target, weight: weight, ignore_index: ignore_index, reduction: reduction)
end
def ctc_loss(log_probs, targets, input_lengths, target_lengths, blank: 0, reduction: "mean", zero_infinity: false)
# call to_a on input_lengths and target_lengths for C++
- Torch._ctc_loss_intlist(log_probs, targets, input_lengths.to_a, target_lengths.to_a, blank, reduction, zero_infinity)
+ Torch.ctc_loss(log_probs, targets, input_lengths.to_a, target_lengths.to_a, blank, reduction, zero_infinity)
end
def hinge_embedding_loss(input, target, margin: 1.0, reduction: "mean")
- Torch._hinge_embedding_loss(input, target, margin, reduction)
+ Torch.hinge_embedding_loss(input, target, margin, reduction)
end
def kl_div(input, target, reduction: "mean")
- Torch._kl_div(input, target, reduction)
+ Torch.kl_div(input, target, reduction)
end
def l1_loss(input, target, reduction: "mean")
- NN._l1_loss(input, target, reduction)
+ NN.l1_loss(input, target, reduction)
end
def margin_ranking_loss(input1, input2, target, margin: 0, reduction: "mean")
raise NotImplementedYet
end
def mse_loss(input, target, reduction: "mean")
- NN._mse_loss(input, target, reduction)
+ NN.mse_loss(input, target, reduction)
end
def multilabel_margin_loss(input, target, reduction: "mean")
- NN._multilabel_margin_loss(input, target, reduction)
+ NN.multilabel_margin_loss(input, target, reduction)
end
def multilabel_soft_margin_loss(input, target, weight: nil)
raise NotImplementedYet
end
def multi_margin_loss(input, target, p: 1, margin: 1.0, weight: nil, reduction: "mean")
- NN._multi_margin_loss(input, target, p, margin, weight, reduction)
+ NN.multi_margin_loss(input, target, p, margin, weight, reduction)
end
def nll_loss(input, target, weight: nil, ignore_index: -100, reduction: "mean")
- NN._nll_loss(input, target, weight, reduction, ignore_index)
+ NN.nll_loss(input, target, weight, reduction, ignore_index)
end
def poisson_nll_loss(input, target, log_input: true, full: false, eps: 1e-8, reduction: "mean")
- Torch._poisson_nll_loss(input, target, log_input, full, eps, reduction)
+ Torch.poisson_nll_loss(input, target, log_input, full, eps, reduction)
end
def soft_margin_loss(input, target, reduction: "mean")
- NN._soft_margin_loss(input, target, reduction)
+ NN.soft_margin_loss(input, target, reduction)
end
def smooth_l1_loss(input, target, reduction: "mean")
- NN._smooth_l1_loss(input, target, reduction)
+ NN.smooth_l1_loss(input, target, reduction)
end
def triplet_margin_loss(anchor, positive, negative, margin: 1.0, p: 2, eps: 1e-06, swap: false, reduction: "mean")
- Torch._triplet_margin_loss(anchor, positive, negative, margin, p, eps, swap, reduction)
- end
-
- # end loss
-
- def softmax(input, dim: nil)
- dim ||= softmax_dim(input.dim)
- input.softmax(dim: dim)
- end
-
- def softmin(input, dim: nil)
- dim ||= softmax_dim(input.dim)
- (-input).softmax(dim: dim)
- end
-
- def softplus(input, beta: 1, threshold: 20)
- NN._softplus(input, beta, threshold)
- end
-
- # TODO make dim keyword argument and update examples
- def log_softmax(input, dim = nil)
- dim ||= softmax_dim(input.dim)
- input.log_softmax(dim)
- end
-
- def dropout(input, p: 0.5, training: true, inplace: false)
- if inplace
- Torch._dropout_(input, p, training)
- else
- Torch._dropout(input, p, training)
- end
- end
-
- def dropout2d(input, p: 0.5, training: true, inplace: false)
- raise ArgumentError, "dropout probability has to be between 0 and 1, but got #{p}" if p < 0 || p > 1
-
- if inplace
- Torch._feature_dropout_(input, p, training)
- else
- Torch._feature_dropout(input, p, training)
- end
- end
-
- def dropout3d(input, p: 0.5, training: true, inplace: false)
- if inplace
- Torch._feature_dropout_(input, p, training)
- else
- Torch._feature_dropout(input, p, training)
- end
- end
-
- def alpha_dropout(input, p: 0.5, training: true, inplace: false)
- if inplace
- Torch._alpha_dropout_(input, p, training)
- else
- Torch._alpha_dropout(input, p, training)
- end
- end
-
- def feature_alpha_dropout(input, p: 0.5, training: true, inplace: false)
- if inplace
- Torch._feature_alpha_dropout_(input, p, training)
- else
- Torch._feature_alpha_dropout(input, p, training)
- end
+ Torch.triplet_margin_loss(anchor, positive, negative, margin, p, eps, swap, reduction)
end
private
def softmax_dim(ndim)