trident
trident
Table Of Contents
trident
Table Of Contents

Source code for trident.layers.pytorch_layers

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import builtins
import inspect
import math
from collections import OrderedDict
from functools import partial, wraps, update_wrapper
from itertools import islice
from itertools import repeat

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F  # import torch functions
import torch.utils.hooks as hooks
import torchvision
from torch._jit_internal import List
from torch._six import container_abcs
from torch.nn import Module
from torch.nn import init
from torch.nn.parameter import Parameter

from trident.layers.pytorch_activations import get_activation
from trident.layers.pytorch_normalizations import *
from trident.backend.common import *
from trident.backend.pytorch_backend import *
from trident.backend.pytorch_ops import *

__all__ = ['Dense', 'Flatten', 'Concatenate', 'Concate','SoftMax','Add', 'Subtract', 'Dot', 'Conv1d', 'Conv2d', 'Conv3d',
           'TransConv1d', 'TransConv2d', 'TransConv3d', 'SeparableConv1d', 'SeparableConv2d', 'SeparableConv3d',
           'DepthwiseConv1d', 'DepthwiseConv2d', 'DepthwiseConv3d', 'GcdConv2d', 'Lambda', 'Reshape',
           'CoordConv2d', 'Upsampling2d', 'Dropout', 'AlphaDropout', 'SelfAttention','SingleImageLayer']

_session = get_session()

_epsilon = _session.epsilon


def _ntuple(n):
    def parse(x):
        if isinstance(x, container_abcs.Iterable):
            return x
        return tuple(repeat(x, n))

    return parse


_single = _ntuple(1)
_pair = _ntuple(2)
_triple = _ntuple(3)
_quadruple = _ntuple(4)


[docs]class Dense(Layer): r"""Applies a linear transformation to the incoming data: :math:`y = xA^T + b` Args: in_features: size of each input sample out_features: size of each output sample bias: If set to ``False``, the layer will not learn an additive bias. Default: ``True`` Shape: - Input: :math:`(N, *, H_{in})` where :math:`*` means any number of additional dimensions and :math:`H_{in} = \text{in\_features}` - Output: :math:`(N, *, H_{out})` where all but the last dimension are the same shape as the input and :math:`H_{out} = \text{out\_features}`. Attributes: weight: the learnable weights of the module of shape :math:`(\text{out\_features}, \text{in\_features})`. The values are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})`, where :math:`k = \frac{1}{\text{in\_features}}` bias: the learnable bias of the module of shape :math:`(\text{out\_features})`. If :attr:`bias` is ``True``, the values are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where :math:`k = \frac{1}{\text{in\_features}}` Examples:: >>> m = Dense(30) >>> input = to_tensor(torch.randn(2, 20)) >>> output = m(input) >>> print(output.size()) torch.Size([2, 30]) """ def __init__(self, output_shape, use_bias=True, activation=None,keep_output=False, name=None, **kwargs): super(Dense, self).__init__() self.rank=0 if isinstance(output_shape, int): self.output_shape = _single(output_shape) elif isinstance(output_shape, list): self.output_shape = tuple(output_shape) elif isinstance(output_shape, tuple): self.output_shape = output_shape else: raise ValueError('output_shape should be integer, list of integer or tuple of integer...') self.name = name self.keep_output=keep_output self.weight = None self.bias = None self.use_bias = use_bias self.activation = get_activation(activation)
[docs] def build(self, input_shape): if self._built == False: self.weight = Parameter(torch.Tensor(*self.output_shape.tolist(), self.input_filters)) init.kaiming_uniform_(self.weight, a=math.sqrt(5)) # self._parameters['weight'] =self.weight if self.use_bias: self.bias = Parameter(torch.Tensor(self.output_shape.tolist()[0])) init.zeros_(self.bias) # self._parameters['bias']=self.bias self.to(self.device) self._built = True
[docs] def forward(self, *x): x = enforce_singleton(x) x = F.linear(x, self.weight, self.bias) if self.activation is not None: x = self.activation(x) return x
[docs] def extra_repr(self): s = 'output_shape={0}'.format(self.output_shape.tolist()) + ',use_bias={use_bias}' if 'activation' in self.__dict__ and self.__dict__['activation'] is not None: if inspect.isfunction(self.__dict__['activation']): s += ', activation={0}'.format(self.__dict__['activation'].__name__) elif isinstance(self.__dict__['activation'], nn.Module): s += ', activation={0}'.format(self.__dict__['activation']).__repr__() return s.format(**self.__dict__)
[docs]class Flatten(Layer): r"""Flatten layer to flatten a tensor after convolution.""" def __init__(self): super(Flatten, self).__init__()
[docs] def forward(self, x: torch.Tensor) -> torch.Tensor: return x.view(x.size(0), -1)
[docs]class Concate(Layer): r"""Concate layer to splice tensors .""" def __init__(self, axis=1): super(Concate, self).__init__() self.axis = axis
[docs] def forward(self, *x) -> torch.Tensor: if not isinstance(x, list) or len(x) < 2: raise ValueError('A `Concatenate` layer should be called on a list of at least 2 inputs') if all([k.size() is None for k in x]): return reduced_inputs_shapes = [list(k.size()) for k in x] shape_set = set() for i in range(len(reduced_inputs_shapes)): del reduced_inputs_shapes[i][self.axis] shape_set.add(tuple(reduced_inputs_shapes[i])) if len(shape_set) > 1: raise ValueError( 'A `Concatenate` layer requires inputs with matching shapes except for the concat axis. Got inputs ' 'shapes: %s' % (shape_set)) x = torch.cat(x, dim=self.axis) return x
Concatenate = Concate
[docs]class Add(Layer): r"""Flatten layer to flatten a tensor after convolution.""" def __init__(self, axis=1): super(Add, self).__init__()
[docs] def build(self, input_shape): if self._built == False: self.output_shape = input_shape self._built = True
[docs] def forward(self, *x) -> torch.Tensor: if not isinstance(x, (list, tuple)): raise ValueError('A merge layer should be called on a list of inputs.') if isinstance(x, tuple): x = unpack_singleton(x) out = 0 for item in x: out = torch.add(out, item) return out
[docs]class Subtract(Layer): r"""Flatten layer to flatten a tensor after convolution.""" def __init__(self, axis=1): super(Subtract, self).__init__()
[docs] def build(self, input_shape): if self._built == False: self.output_shape = input_shape self._built = True
[docs] def forward(self, *x) -> torch.Tensor: if not isinstance(x, (list, tuple)): raise ValueError('A merge layer should be called on a list of inputs.') if not isinstance(x, tuple): x = list(x) out = 0 for item in x: out = torch.sub(out, item) return out
[docs]class Dot(Layer): r"""Flatten layer to flatten a tensor after convolution.""" def __init__(self, axis=1): super(Dot, self).__init__()
[docs] def build(self, input_shape): if self._built == False: self.output_shape = input_shape self._built = True
[docs] def forward(self, *x) -> torch.Tensor: if not isinstance(x, (list, tuple)): raise ValueError('A merge layer should be called on a list of inputs.') if not isinstance(x, tuple): x = list(x) out = 0 for item in x: out = torch.dot(out, item) return out
[docs]class SoftMax(Layer): r"""SoftMax layer to accelerate classification model training""" def __init__(self, axis=1, add_noise=False, noise_intensity=0.005, **kwargs): super(SoftMax, self).__init__() self.axis = kwargs.get('dim', axis)
[docs] def forward(self, *x) -> torch.Tensor: x = enforce_singleton(x) if not hasattr(self, 'add_noise'): self.add_noise = False self.noise_intensity = 0.005 if self.training: if self.add_noise == True: noise = self.noise_intensity * torch.randn_like(x, dtype=torch.float32) x = x + noise x = F.log_softmax(x, dim=1) else: x = torch.softmax(x, dim=1) return x
_gcd = gcd _get_divisors = get_divisors _isprime = isprime def get_static_padding(rank,kernal_shape,strides,dilations,input_shape=None,transpose=False): ''' Calcualte the actual padding we need in different rank and different convlution settings. Args: rank (int): kernal_shape (tuple of integer): strides (tuple of integer): dilations (tuple of integer): input_shape (None or tuple of integer): transpose (bool): wheather transposed Returns: the padding we need (shape: 2*rank ) Examples >>> get_static_padding(1,(3,),(2,),(2,)) (2, 2) >>> get_static_padding(2,(3,3),(2,2),(1,1),(224,224)) (1, 1, 1, 1) >>> get_static_padding(2,(3,3),(2,2),(1,1),(224,224),True) ((1, 1, 1, 1), (1, 1)) >>> get_static_padding(2,(5,5),(1,1),(2,2)) (4, 4, 4, 4) >>> get_static_padding(2,(5,5),(1,1),(1,1)) (2, 2, 2, 2) >>> get_static_padding(2,(2,2),(1,1),(1,1)) (1, 0, 1, 0) >>> get_static_padding(3,(5,5,5),(1,1,1),(2,2,2)) (4, 4, 4, 4, 4, 4) ''' if input_shape is None: input_shape=[224]*rank if isinstance(kernal_shape,int): kernal_shape= _ntuple(rank)(kernal_shape) if isinstance(strides,int): strides= _ntuple(rank)(strides) if isinstance(dilations,int): dilations= _ntuple(rank)(dilations) input_shape=to_numpy(input_shape) kernal_shape=to_numpy(list(kernal_shape)) strides = to_numpy(list(strides)).astype(np.float32) dilations= to_numpy(list(dilations)) if transpose==False: output_shape=np.ceil(input_shape/strides) raw_padding=np.clip((output_shape-1)*strides+(kernal_shape-1)*dilations+1-input_shape,a_min=0,a_max=np.inf) remainder=np.remainder(raw_padding,np.ones_like(raw_padding)*2) raw_padding=raw_padding+(remainder*np.greater(strides,1).astype(np.float32)) lefttop_pad = np.ceil(raw_padding/2.0).astype(np.int32) rightbtm_pad=(raw_padding-lefttop_pad).astype(np.int32) static_padding = [] for k in range(rank): static_padding.append(lefttop_pad[-1-k]) static_padding.append(rightbtm_pad[-1-k]) return static_padding else: output_shape = input_shape * strides raw_padding = np.clip( ((input_shape - 1) * strides + (kernal_shape - 1) * dilations + 1 )-input_shape * strides, a_min=0, a_max=np.inf) remainder = np.remainder(raw_padding, np.ones_like(raw_padding) * 2) raw_padding= raw_padding + (remainder * np.greater(strides, 1).astype(np.float32)) lefttop_pad = np.ceil(raw_padding / 2.0).astype(np.int32) rightbtm_pad = (raw_padding- lefttop_pad).astype(np.int32) out_pad=output_shape-((input_shape - 1) * strides + (kernal_shape - 1) * dilations + 1-lefttop_pad-rightbtm_pad) static_padding = [] for k in range(rank): static_padding.append(lefttop_pad[-1 - k]) static_padding.append(rightbtm_pad[-1 - k]) return tuple(static_padding), tuple(out_pad.astype(np.int32).tolist()) class _ConvNd(Layer): __constants__ = ['kernel_size', 'num_filters', 'strides', 'auto_pad', 'padding_mode', 'use_bias', 'dilation', 'groups', 'transposed'] def __init__(self, rank,kernel_size, num_filters, strides, auto_pad,padding, padding_mode, use_bias, dilation, groups, transposed=False, name=None, depth_multiplier=1, depthwise=False,separable=False,**kwargs): super(_ConvNd, self).__init__(name=name) self.rank=rank self.num_filters = num_filters self.depth_multiplier=depth_multiplier self.kernel_size = kernel_size self.strides = strides self.dilation = dilation self.transposed = transposed if self.transposed: self.output_padding=_ntuple(rank)(0) self.groups = groups self.auto_pad = auto_pad self.padding_mode = padding_mode if padding is not None: self.padding = normalize_padding(padding,rank) else: self.padding=None self.depthwise = depthwise self.separable = separable if self.separable == True: self.depthwise = True self.register_parameter('weight',None) self.register_parameter('bias', None) self.transposed = transposed self.use_bias = use_bias self.to(self.device) def build(self, input_shape): if self._built == False: self.input_filters=input_shape[0].item() if self.auto_pad: if self.transposed==False: padding = get_static_padding(self.rank, self.kernel_size, self.strides, self.dilation,input_shape.tolist()[1:]) self.padding=tuple(padding) else: self.padding ,self.output_padding= get_static_padding(self.rank, self.kernel_size, self.strides, self.dilation, input_shape.tolist()[1:], self.transposed) else: if self.padding is None: self.padding = [0] * (2 * self.rank) elif isinstance(self.padding, int): self.self.padding = [self.padding] * (2 * self.rank) elif len(self.padding) == self.rank: self.padding = list(self.padding)*2 elif len(self.padding) == 2 * self.rank: pass if self.depthwise or self.separable: if self.depth_multiplier is None: self.depth_multiplier = 1 if self.groups>1: pass elif self.depth_multiplier<1: self.groups=int(builtins.round(self.input_filters*self.depth_multiplier,0)) else: self.groups=self.input_filters if self.groups==1 else self.groups if self.num_filters is None and self.depth_multiplier is not None: self.num_filters = int(builtins.round(self.input_filters * self.depth_multiplier,0)) if self.groups != 1 and self.num_filters % self.groups != 0: raise ValueError('out_channels must be divisible by groups') if self.depthwise and self.num_filters % self.groups != 0: raise ValueError('out_channels must be divisible by groups') channel_multiplier = int(self.num_filters // self.groups) if self.depth_multiplier is None else self.depth_multiplier# default channel_multiplier if self.transposed: self.weight = Parameter(torch.Tensor(int(self.input_filters), int(self.num_filters // self.groups), *self.kernel_size)) else: self.weight = Parameter(torch.Tensor(int(self.num_filters), int(self.input_filters // self.groups) , *self.kernel_size)) # if self.separable: self.pointwise = Parameter(torch.Tensor(int(self.input_filters * self.depth_multiplier), int(self.num_filters),1,1)) init.kaiming_uniform_(self.weight, a=math.sqrt(5)) if self.use_bias: self.bias = Parameter(torch.Tensor(int(self.num_filters))) init.zeros_(self.bias) self.to(get_device()) self._built = True def extra_repr(self): s = 'kernel_size={kernel_size}, num_filters={num_filters},strides={strides}' if 'activation' in self.__dict__ and self.__dict__['activation'] is not None: if inspect.isfunction(self.__dict__['activation']): s += ', activation={0}'.format(self.__dict__['activation'].__name__) elif isinstance(self.__dict__['activation'], nn.Module): s += ', activation={0}'.format(self.__dict__['activation']).__repr__() s += ',auto_pad={auto_pad},use_bias={use_bias} ,dilation={dilation}' if self.groups != 1: s += ', groups={groups}' if self._input_shape is not None: s += ', input_shape={0}, input_filter={1}'.format(to_numpy(self._input_shape).tolist(), self.input_filters) if self.output_shape is not None: s += ', output_shape={0}'.format(self.output_shape if isinstance(self.output_shape, ( list, tuple)) else self.output_shape.clone().tolist()) # if self.bias is None: # s += ', use_bias=False' return s.format(**self.__dict__) def __setstate__(self, state): super(_ConvNd, self).__setstate__( state) # if not hasattr(self, 'padding_mode'): # self.padding_mode = 'zeros'
[docs]class Conv1d(_ConvNd): r"""Applies to create a 1D convolution layer Args: kernel_size :(int or tupleof ints) shape (spatial extent) of the receptive field num_filters :(int or None, default to None) number of output channel (filters)`, sometimes in backbond design output channel is propotional to input channel. But in trident all layer is shape delay inferred strides:(int or tupleof ints ,default to 1) stride of the convolution (increment when sliding the filter over the input) auto_pad:bool if `False`, then the filter will be shifted over the "valid" area of input, that is, no value outside the area is used. If ``pad=True`` means 'same *padding (optional) auto_pad can help you calculate the pad you need. if you have special need , you still can use the paddding implicit paddings on both sides of the input. Can be a single number or a double tuple (padH, padW) or quadruple(pad_left, pad_right, pad_top, pad_btm ) padding_mode:string (default is 'zero', available option are 'reflect', 'replicate','constant','circular') activation: (None, string, function or Layer) activation function after the convolution operation for apply non-linearity. use_bias:bool the layer will have no bias if `False` is passed here dilation:(int or tupleof ints) the spacing between kernel elements. Can be a single number or a tuple (dH, dW). Default: 1 groups split input into groups, \text{in\_channels}in_channels should be divisible by the number of groups. Default: 1 depth_multiplier: (int of decimal) name name of the layer Shape: - Input: :math:`(N, *, H_{in})` where :math:`*` means any number of additional dimensions and :math:`H_{in} = \text{in\_features}` - Output: :math:`(N, *, H_{out})` where all but the last dimension are the same shape as the input and :math:`H_{out} = \text{out\_features}`. Attributes: weight: the learnable weights of the module of shape :math:`(\text{out\_features}, \text{in\_features})`. The values are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})`, where :math:`k = \frac{1}{\text{in\_features}}` bias: the learnable bias of the module of shape :math:`(\text{out\_features})`. If :attr:`bias` is ``True``, the values are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where :math:`k = \frac{1}{\text{in\_features}}` Examples:: >>> input = to_tensor(torch.randn(1,64,32)) >>> conv1= Conv1d(3,64,strides=2,activation='leaky_relu', auto_pad=True,use_bias=False) >>> output = conv1(input) >>> print(output.size()) torch.Size([1, 64, 16]) >>> print(conv1.weight.size()) torch.Size([64, 64, 3]) >>> print(conv1.padding) (1, 1) >>> conv2= Conv1d(3, 256, strides=2, auto_pad=False, padding=1) >>> output = conv2(input) >>> print(output.size()) torch.Size([1, 256, 16]) >>> print(conv2.weight.size()) torch.Size([256, 64, 3]) >>> print(conv2.padding) (1, 1) >>> conv3= Conv1d(5,64,strides=1,activation=mish, auto_pad=True,use_bias=False,dilation=4,groups=16) >>> output = conv3(input) >>> print(output.size()) torch.Size([1, 64, 32]) >>> print(conv3.weight.size()) torch.Size([64, 4, 5]) >>> print(conv3.padding) (8, 8) >>> input = to_tensor(torch.randn(1,32,37)) >>> conv4= Conv1d(3,64,strides=2,activation=mish, auto_pad=True,use_bias=False) >>> output = conv4(input) >>> print(output.size()) torch.Size([1, 64, 19]) """ def __init__(self, kernel_size, num_filters=None, strides=1, auto_pad=True, padding=None,padding_mode='zero', activation=None, use_bias=False, dilation=1, groups=1, name=None, depth_multiplier=None, **kwargs): rank = 1 kernel_size = _single(kernel_size) strides = _single(kwargs.get('stride',strides)) dilation = _single(kwargs.get('dilation_rate',dilation)) num_filters=kwargs.get('filters',kwargs.get('out_channels',num_filters)) use_bias=kwargs.get('bias',use_bias) padding_mode = padding_mode.lower().replace('zeros','zero') if isinstance(padding_mode, str) else padding_mode if isinstance(padding, str) and auto_pad==False: #avoid someone use the definition as keras padding auto_pad = (padding.lower() == 'same') auto_pad = False elif isinstance(padding, int): padding = _single(padding) auto_pad = False elif isinstance(padding, tuple): pass super(Conv1d, self).__init__(rank,kernel_size, num_filters, strides, auto_pad,padding,padding_mode, use_bias, dilation, groups, transposed=False, name=name, depth_multiplier=depth_multiplier,depthwise=False,separable=False, **kwargs) self.activation = get_activation(activation)
[docs] def conv1d_forward(self, x): x = F.pad(x, self.padding, mode='constant' if self.padding_mode == 'zero' else self.padding_mode) return F.conv1d(x, self.weight, self.bias, self.strides, _single(0), self.dilation, self.groups)
[docs] def forward(self, *x): x = enforce_singleton(x) x = self.conv1d_forward(x) if self.activation is not None: x = self.activation(x) return x
[docs]class Conv2d(_ConvNd): r"""Applies to create a 2D convolution layer Args: kernel_size :(int or tupleof ints) shape (spatial extent) of the receptive field num_filters :(int or None, default to None) number of output channel (filters)`, sometimes in backbond design output channel is propotional to input channel. But in trident all layer is shape delay inferred strides:(int or tupleof ints ,default to 1) stride of the convolution (increment when sliding the filter over the input) auto_pad:bool if `False`, then the filter will be shifted over the "valid" area of input, that is, no value outside the area is used. If ``pad=True`` means 'same *padding (optional) auto_pad can help you calculate the pad you need. if you have special need , you still can use the paddding implicit paddings on both sides of the input. Can be a single number or a double tuple (padH, padW) or quadruple(pad_left, pad_right, pad_top, pad_btm ) padding_mode:string (default is 'zero', available option are 'reflect', 'replicate','constant','circular') activation: (None, string, function or Layer) activation function after the convolution operation for apply non-linearity. use_bias:bool the layer will have no bias if `False` is passed here dilation:(int or tupleof ints) the spacing between kernel elements. Can be a single number or a tuple (dH, dW). Default: 1 groups split input into groups, \text{in\_channels}in_channels should be divisible by the number of groups. Default: 1 depth_multiplier: (int of decimal) name name of the layer Shape: - Input: :math:`(N, *, H_{in})` where :math:`*` means any number of additional dimensions and :math:`H_{in} = \text{in\_features}` - Output: :math:`(N, *, H_{out})` where all but the last dimension are the same shape as the input and :math:`H_{out} = \text{out\_features}`. Attributes: weight: the learnable weights of the module of shape :math:`(\text{out\_features}, \text{in\_features})`. The values are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})`, where :math:`k = \frac{1}{\text{in\_features}}` bias: the learnable bias of the module of shape :math:`(\text{out\_features})`. If :attr:`bias` is ``True``, the values are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where :math:`k = \frac{1}{\text{in\_features}}` Examples:: >>> input = to_tensor(torch.randn(1,32,32,32)) >>> conv1= Conv2d((3,3),64,strides=2,activation='leaky_relu', auto_pad=True,use_bias=False) >>> output = conv1(input) >>> print(output.size()) torch.Size([1, 64, 16, 16]) >>> print(conv1.weight.size()) torch.Size([64, 32, 3, 3]) >>> print(conv1.padding) (1, 1, 1, 1) >>> conv2= Conv2d((3, 3), 256, strides=(2, 2), auto_pad=False, padding=((1, 0), (1, 0))) >>> output = conv2(input) >>> print(output.size()) torch.Size([1, 256, 16, 16]) >>> print(conv2.weight.size()) torch.Size([256, 32, 3, 3]) >>> print(conv2.padding) (1, 0, 1, 0) >>> conv3= Conv2d((3,5),64,strides=(1,2),activation=mish, auto_pad=True,use_bias=False,dilation=4,groups=16) >>> output = conv3(input) >>> print(output.size()) torch.Size([1, 64, 32, 16]) >>> print(conv3.weight.size()) torch.Size([64, 2, 3, 5]) >>> print(conv3.padding) (8, 8, 4, 4) >>> input = to_tensor(torch.randn(1,32,37,37)) >>> conv4= Conv2d((3,3),64,strides=2,activation=mish, auto_pad=True,use_bias=False) >>> output = conv4(input) >>> print(output.size()) torch.Size([1, 64, 19, 19]) """ def __init__(self, kernel_size, num_filters=None, strides=1, auto_pad=True, padding=None,padding_mode='zero', activation=None, use_bias=False, dilation=1, groups=1, name=None, depth_multiplier=None, **kwargs): rank = 2 kernel_size = _pair(kernel_size) strides = _pair(kwargs.get('stride', strides)) dilation = _pair(kwargs.get('dilation_rate', dilation)) num_filters = kwargs.get('filters', kwargs.get('out_channels', num_filters)) use_bias = kwargs.get('bias', use_bias) padding_mode = padding_mode.lower().replace('zeros', 'zero') if isinstance(padding_mode, str) else padding_mode if isinstance(padding, str) and auto_pad==False: auto_pad = (padding.lower() == 'same') padding=None elif isinstance(padding, int) and padding>0: padding = _pair(padding) auto_pad = False elif isinstance(padding, tuple): auto_pad = False pass super(Conv2d, self).__init__(rank, kernel_size, num_filters, strides, auto_pad, padding, padding_mode, use_bias, dilation, groups, transposed=False, name=name, depth_multiplier=depth_multiplier, depthwise=False, separable=False, **kwargs) self.activation = get_activation(activation)
[docs] def conv2d_forward(self, x): if len(self.padding)==self.rank: self.padding=(self.padding[1],self.padding[1],self.padding[0],self.padding[0]) if self.padding_mode == 'circular': expanded_padding = ((self.padding[0] + 1) // 2, self.padding[1] // 2, (self.padding[2] + 1) // 2, self.padding[3] // 2) x = F.pad(x, expanded_padding, mode='circular') else: x = F.pad(x, self.padding,mode='constant' if self.padding_mode == 'zero' else self.padding_mode) return F.conv2d(x, self.weight, self.bias, self.strides, _pair(0), self.dilation, self.groups)
[docs] def forward(self, *x): x = enforce_singleton(x) x = self.conv2d_forward(x) if self.activation is not None: x = self.activation(x) return x
[docs]class Conv3d(_ConvNd): def __init__(self, kernel_size, num_filters=None, strides=1, auto_pad=True, padding=None,padding_mode='zero', activation=None, use_bias=False, dilation=1, groups=1, name=None, depth_multiplier=None, **kwargs): rank = 3 kernel_size = _triple(kernel_size) strides = _triple(kwargs.get('stride', strides)) dilation = _triple(kwargs.get('dilation_rate', dilation)) num_filters = kwargs.get('filters', kwargs.get('out_channels', num_filters)) use_bias = kwargs.get('bias', use_bias) padding_mode = padding_mode.lower().replace('zeros', 'zero') if isinstance(padding_mode, str) else padding_mode if isinstance(padding, str)and auto_pad==False: auto_pad = (padding.lower() == 'same') elif isinstance(padding, int): padding = _triple(padding) elif isinstance(padding, tuple): pass super(Conv3d, self).__init__(rank, kernel_size, num_filters, strides, auto_pad, padding, padding_mode, use_bias, dilation, groups, transposed=False, name=name, depth_multiplier=depth_multiplier, depthwise=False, separable=False, **kwargs) self.activation = get_activation(activation)
[docs] def conv3d_forward(self, x): if self.padding_mode == 'circular': expanded_padding = ((self.padding[2] + 1) // 2, self.padding[2] // 2, (self.padding[1] + 1) // 2, self.padding[1] // 2,(self.padding[0] + 1) // 2, self.padding[0] // 2) x = F.pad(x, expanded_padding, mode='circular') else: x = F.pad(x, (self.padding[2], self.padding[2],self.padding[1], self.padding[1], self.padding[0], self.padding[0]),mode='constant' if self.padding_mode == 'zero' else self.padding_mode) return F.conv3d(x, self.weight, self.bias, self.strides, _triple(0), self.dilation, self.groups)
[docs] def forward(self, *x): x = enforce_singleton(x) x = self.conv3d_forward(x) if self.activation is not None: x = self.activation(x) return x
[docs]class TransConv1d(_ConvNd): def __init__(self, kernel_size, num_filters=None, strides=1, auto_pad=True, padding=None,padding_mode='zero', activation=None, use_bias=False, dilation=1, groups=1, name=None, depth_multiplier=None, **kwargs): rank = 1 kernel_size = _single(kernel_size) strides = _single(kwargs.get('stride', strides)) dilation = _single(kwargs.get('dilation_rate', dilation)) num_filters = kwargs.get('filters', kwargs.get('out_channels', num_filters)) use_bias = kwargs.get('bias', use_bias) padding_mode = padding_mode.lower().replace('zeros', 'zero') if isinstance(padding_mode, str) else padding_mode if isinstance(padding, str)and auto_pad==False: auto_pad = (padding.lower() == 'same') elif isinstance(padding, int): padding = _single(padding) elif isinstance(padding, tuple): pass super(TransConv1d, self).__init__(rank, kernel_size, num_filters, strides, auto_pad, padding, padding_mode, use_bias, dilation, groups, transposed=True, name=name, depth_multiplier=depth_multiplier, depthwise=False, separable=False, **kwargs) self.activation = get_activation(activation) self.output_padding = _single(0)
[docs] def conv1d_forward(self, x): x = F.pad(x, self.padding, mode='constant' if self.padding_mode == 'zero' else self.padding_mode) if self.padding>0: self.output_padding = _single(1) return F.conv_transpose1d(x, self.weight, self.bias, self.strides, padding=_single(0),output_padding=self.output_padding, dilation=self.dilation, groups=self.groups)
[docs] def forward(self, *x): x = enforce_singleton(x) x = self.conv1d_forward(x) if self.activation is not None: x = self.activation(x) return x
[docs]class TransConv2d(_ConvNd): ''' Examples: >>> input = to_tensor(torch.randn(1,32,128,128)) >>> conv1= TransConv2d((3,3),64,strides=2,activation='leaky_relu', auto_pad=True,use_bias=False) >>> output = conv1(input) >>> conv1.padding (1, 1, 1, 1) >>> conv1.output_padding (1, 1) >>> print(output.size()) torch.Size([1, 64, 256, 256]) ''' def __init__(self, kernel_size, num_filters=None, strides=1, auto_pad=True, padding=None,padding_mode='zero', activation=None, use_bias=False, dilation=1, groups=1, name=None, depth_multiplier=None, **kwargs): rank = 2 kernel_size = _pair(kernel_size) strides = _pair(kwargs.get('stride', strides)) dilation = _pair(kwargs.get('dilation_rate', dilation)) num_filters = kwargs.get('filters', kwargs.get('out_channels', num_filters)) use_bias = kwargs.get('bias', use_bias) padding_mode = padding_mode.lower().replace('zeros', 'zero') if isinstance(padding_mode, str) else padding_mode if isinstance(padding, str)and auto_pad==False: auto_pad = (padding.lower() == 'same') elif isinstance(padding, int): padding = _pair(padding) elif isinstance(padding, tuple): pass super(TransConv2d, self).__init__(rank, kernel_size, num_filters, strides, auto_pad, padding, padding_mode, use_bias, dilation, groups, transposed=True, name=name, depth_multiplier=depth_multiplier, depthwise=False, separable=False, **kwargs) self.activation = get_activation(activation) self.output_padding = _pair(0) # def get_padding(self, input_shape): # pad_h = 0 # pad_w = 0 # if self.auto_pad == True: # ih, iw = list(input_shape)[-2:] # kh, kw = self.kernel_size[-2:] # sh, sw = self.strides[-2:] # dh, dw = self.dilation[-2:] # oh, ow = (ih - 1) * sh + (kh - 1) * dh + 1, (iw - 1) * sw + (kw - 1) * dw + 1 # pad_h = max(oh - ih * sh, 0) # pad_w = max(ow - iw * sw, 0) # self.padding = (pad_h, pad_w) # if pad_h != 0 or pad_w != 0: # self.output_padding = (pad_h % 2 if pad_h > 0 else pad_h, pad_w % 2 if pad_w > 0 else pad_w)
[docs] def conv2d_forward(self, x): # if len(self.padding) == self.rank: # self.padding = (self.padding[1], self.padding[1], self.padding[0], self.padding[0]) # if self.padding_mode == 'circular': # expanded_padding = ( # (self.padding[0] + 1) // 2, self.padding[1] // 2, (self.padding[2] + 1) // 2, self.padding[3] // 2) # x = F.pad(x, expanded_padding, mode='circular') # else: # x = F.pad(x, self.padding, mode='constant' if self.padding_mode == 'zero' else self.padding_mode) return F.conv_transpose2d(x, self.weight, self.bias, self.strides, padding=(self.padding[0],self.padding[2]), output_padding=self.output_padding, dilation=self.dilation, groups=self.groups)
[docs] def forward(self, *x): x = enforce_singleton(x) x = self.conv2d_forward(x) if self.activation is not None: x = self.activation(x) return x
[docs]class TransConv3d(_ConvNd): def __init__(self, kernel_size, num_filters=None, strides=1, auto_pad=True, padding=None,padding_mode='zero', activation=None, use_bias=False, dilation=1, groups=1, name=None, depth_multiplier=None, **kwargs): rank = 3 kernel_size = _triple(kernel_size) strides = _triple(kwargs.get('stride', strides)) dilation = _triple(kwargs.get('dilation_rate', dilation)) num_filters = kwargs.get('filters', kwargs.get('out_channels', num_filters)) use_bias = kwargs.get('bias', use_bias) padding_mode = padding_mode.lower().replace('zeros', 'zero') if isinstance(padding_mode, str) else padding_mode if isinstance(padding, str)and auto_pad==False: auto_pad = (padding.lower() == 'same') elif isinstance(padding, int): padding = _triple(padding) elif isinstance(padding, tuple): pass super(TransConv3d, self).__init__(rank, kernel_size, num_filters, strides, auto_pad, padding, padding_mode, use_bias, dilation, groups, transposed=True, name=name, depth_multiplier=depth_multiplier, depthwise=False, separable=False, **kwargs) self.activation = get_activation(activation) self.output_padding = _triple(0)
[docs] def conv3d_forward(self, x): if self.auto_pad == True: iz, ih, iw = list(x.size())[-3:] kz, kh, kw = self.kernel_size[-3:] sz, sh, sw = self.strides[-3:] dz, dh, dw = self.dilation[-3:] oz, oh, ow = math.ceil(iz / sz), math.ceil(ih / sh), math.ceil(iw / sw) pad_z = max((oz - 1) * sz + (kz - 1) * dz + 1 - iz, 0) pad_h = max((oh - 1) * sh + (kh - 1) * dh + 1 - ih, 0) pad_w = max((ow - 1) * sw + (kw - 1) * dw + 1 - iw, 0) if pad_z > 0 or pad_h > 0 or pad_w > 0: self.output_padding = _triple(1) x = F.pad(x, [pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2, pad_z // 2, pad_z - pad_z // 2], mode=self.padding_mode) return F.conv_transpose3d(x, self.weight, self.bias, self.strides, padding=_triple(0), output_padding=self.output_padding, dilation=self.dilation, groups=self.groups)
[docs] def forward(self, *x): x = enforce_singleton(x) x = self.conv3d_forward(x) if self.activation is not None: x = self.activation(x) return x
[docs]class SeparableConv1d(_ConvNd): def __init__(self, kernel_size,num_filters=None, depth_multiplier=1, strides=1, auto_pad=True,padding=None, padding_mode='zero', activation=None, use_bias=False, dilation=1, groups=1, name=None, **kwargs): rank = 1 kernel_size = _single(kernel_size) strides = _single(kwargs.get('stride', strides)) dilation = _single(kwargs.get('dilation_rate', dilation)) num_filters = kwargs.get('filters', kwargs.get('out_channels', num_filters)) use_bias = kwargs.get('bias', use_bias) padding_mode = padding_mode.lower().replace('zeros', 'zero') if isinstance(padding_mode, str) else padding_mode if isinstance(padding, str)and auto_pad==False: auto_pad = (padding.lower() == 'same') elif isinstance(padding, int): padding = _single(padding) elif isinstance(padding, tuple): pass super(SeparableConv1d, self).__init__(rank, kernel_size, num_filters, strides, auto_pad, padding, padding_mode, use_bias, dilation, groups, transposed=False, name=name, depth_multiplier=depth_multiplier, depthwise=True, separable=True, **kwargs) self.activation = get_activation(activation) self.conv1 = None self.pointwise = None
[docs] def forward(self, *x): x = enforce_singleton(x) x = self.conv1(x) x = self.pointwise(x) if self.activation is not None: x = self.activation(x) return x
[docs]class SeparableConv2d(_ConvNd): def __init__(self, kernel_size, num_filters=None, depth_multiplier=1, strides=1, auto_pad=True, padding=None,padding_mode='zero', activation=None, use_bias=False, dilation=1, groups=1, name=None, **kwargs): rank = 2 kernel_size = _pair(kernel_size) strides = _pair(kwargs.get('stride', strides)) dilation = _pair(kwargs.get('dilation_rate', dilation)) num_filters = kwargs.get('filters', kwargs.get('out_channels', num_filters)) use_bias = kwargs.get('bias', use_bias) padding_mode = padding_mode.lower().replace('zeros', 'zero') if isinstance(padding_mode, str) else padding_mode if isinstance(padding, str)and auto_pad==False: auto_pad = (padding.lower() == 'same') elif isinstance(padding, int): padding = _pair(padding) elif isinstance(padding, tuple): pass super(SeparableConv2d, self).__init__(rank, kernel_size, num_filters, strides, auto_pad, padding, padding_mode, use_bias, dilation, groups, transposed=False, name=name, depth_multiplier=depth_multiplier, depthwise=True, separable=True, **kwargs) self.activation = get_activation(activation) self.pointwise = None self._built = False
[docs] def build(self, input_shape): if self._built == False or self.conv1 is None: if self.num_filters is None: self.num_filters = self.input_filters * self.depth_multiplier if self.depth_multiplier is not None else self.num_filters self.conv1 = DepthwiseConv2d(kernel_size=self.kernel_size, depth_multiplier=self.depth_multiplier, strides=self.strides, auto_pad=self.auto_pad, padding_mode=self.padding_mode, activation=self.activation, dilation=self.dilation, use_bias=self.use_bias) self.pointwise = Conv2d(kernel_size=(1, 1), num_filters=self.num_filters, strides=1, use_bias=self.use_bias, dilation=1, groups=1) self.to(self.device) self._built = True
[docs] def forward(self, *x): x = enforce_singleton(x) x = self.conv1(x) x = self.pointwise(x) return x
[docs]class SeparableConv3d(_ConvNd): def __init__(self, kernel_size, num_filters=None,depth_multiplier=1, strides=1, auto_pad=True,padding=None, padding_mode='zero', activation=None, use_bias=False, dilation=1, groups=1, name=None, **kwargs): rank = 3 kernel_size = _triple(kernel_size) strides = _triple(kwargs.get('stride', strides)) dilation = _triple(kwargs.get('dilation_rate', dilation)) num_filters = kwargs.get('filters', kwargs.get('out_channels', num_filters)) use_bias = kwargs.get('bias', use_bias) padding_mode = padding_mode.lower().replace('zeros', 'zero') if isinstance(padding_mode, str) else padding_mode if isinstance(padding, str)and auto_pad==False: auto_pad = (padding.lower() == 'same') elif isinstance(padding, int): padding = _triple(padding) elif isinstance(padding, tuple): pass super(SeparableConv3d, self).__init__(rank, kernel_size, num_filters, strides, auto_pad, padding, padding_mode, use_bias, dilation, groups, transposed=False, name=name, depth_multiplier=depth_multiplier, depthwise=True, separable=True, **kwargs) self.activation = get_activation(activation) self.pointwise = None
[docs] def build(self, input_shape): if self._built == False or self.conv1 is None: self.num_filters = self.input_filters * self.depth_multiplier if self.depth_multiplier is not None else self.num_filters self.conv1 = DepthwiseConv3d(kernel_size=self.kernel_size, depth_multiplier=self.depth_multiplier, strides=self.strides, auto_pad=self.auto_pad, padding_mode=self.padding_mode, dilation=self.dilation, groups=self.input_filters, bias=self.use_bias) self.pointwise = Conv3d(kernel_size=(1, 1, 1), depth_multiplier=1, strides=1, use_bias=self.use_bias, dilation=1, groups=1) self.to(self.device) self._built = True
[docs] def forward(self, *x): x = enforce_singleton(x) x = self.conv1(x) x = self.pointwise(x) if self.activation is not None: x = self.activation(x) return x
[docs]class DepthwiseConv1d(_ConvNd): def __init__(self, kernel_size, depth_multiplier=1, strides=1, auto_pad=True,padding=None, padding_mode='zero', activation=None, use_bias=False, dilation=1, name=None, **kwargs): rank = 1 kernel_size = _single(kernel_size) strides = _single(kwargs.get('stride', strides)) dilation = _single(kwargs.get('dilation_rate', dilation)) num_filters = kwargs.get('filters', kwargs.get('out_channels', kwargs.get('num_filters', None))) use_bias = kwargs.get('bias', use_bias) padding_mode = padding_mode.lower().replace('zeros', 'zero') if isinstance(padding_mode, str) else padding_mode if isinstance(padding, str) and auto_pad==False: auto_pad = (padding.lower() == 'same') elif isinstance(padding, int): padding = _single(padding) elif isinstance(padding, tuple): pass groups=1 super(DepthwiseConv1d, self).__init__(rank, kernel_size, num_filters, strides, auto_pad, padding, padding_mode, use_bias, dilation, groups, transposed=False, name=name, depth_multiplier=depth_multiplier, depthwise=True, separable=False, **kwargs) self.activation = get_activation(activation)
[docs] def conv1d_forward(self, x): x = F.pad(x, self.padding, mode='constant' if self.padding_mode == 'zero' else self.padding_mode) return F.conv1d(x, self.weight, self.bias, self.strides, _single(0), self.dilation, self.groups)
[docs] def forward(self, *x): x = self.conv1d_forward(x) if self.activation is not None: x = self.activation(x) return x
[docs]class DepthwiseConv2d(_ConvNd): r"""Applies to create a 2D Depthwise convolution layer Depthwise convolution performs just the first step of a depthwise spatial convolution (which acts on each input channel separately). Args: kernel_size :(int or tupleof ints) shape (spatial extent) of the receptive field depth_multiplier:(int , decimal or None, default to None) The number of depthwise convolution output filters for each input filters. The total number of depthwise convolution output filters will be equal to input_filters * depth_multiplier strides:(int or tupleof ints ,default to 1) stride of the convolution (increment when sliding the filter over the input) auto_pad:bool if `False`, then the filter will be shifted over the "valid" area of input, that is, no value outside the area is used. If ``pad=True`` means 'same *padding (optional) auto_pad can help you calculate the pad you need. if you have special need , you still can use the paddding implicit paddings on both sides of the input. Can be a single number or a double tuple (padH, padW) or quadruple(pad_left, pad_right, pad_top, pad_btm ) padding_mode:string (default is 'zero', available option are 'reflect', 'replicate','constant','circular') activation: (None, string, function or Layer) activation function after the convolution operation for apply non-linearity. use_bias:bool the layer will have no bias if `False` is passed here dilation:(int or tupleof ints) the spacing between kernel elements. Can be a single number or a tuple (dH, dW). Default: 1 groups split input into groups, \text{in\_channels}in_channels should be divisible by the number of groups. Default: 1 name name of the layer Shape: - Input: :math:`(N, *, H_{in})` where :math:`*` means any number of additional dimensions and :math:`H_{in} = \text{in\_features}` - Output: :math:`(N, *, H_{out})` where all but the last dimension are the same shape as the input and :math:`H_{out} = \text{out\_features}`. Attributes: weight: the learnable weights of the module of shape :math:`(\text{out\_features}, \text{in\_features})`. The values are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})`, where :math:`k = \frac{1}{\text{in\_features}}` bias: the learnable bias of the module of shape :math:`(\text{out\_features})`. If :attr:`bias` is ``True``, the values are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where :math:`k = \frac{1}{\text{in\_features}}` Examples:: >>> input = to_tensor(torch.randn(1,32,32,32)) >>> conv1= DepthwiseConv2d((3,3),depth_multiplier=2,strides=2,activation='leaky_relu', auto_pad=True,use_bias=False) >>> output = conv1(input) >>> print(output.size()) torch.Size([1, 64, 16, 16]) >>> print(conv1.weight.size()) torch.Size([64, 1, 3, 3]) >>> print(conv1.padding) (1, 1, 1, 1) >>> print(conv1.num_filters) 64 """ def __init__(self, kernel_size, depth_multiplier=1, strides=1, auto_pad=True, padding=None,padding_mode='zero', activation=None, use_bias=False, dilation=1, name=None, **kwargs): rank = 2 kernel_size = _pair(kernel_size) strides = _pair(kwargs.get('stride', strides)) dilation = _pair(kwargs.get('dilation_rate', dilation)) num_filters =kwargs.get('filters', kwargs.get('out_channels', kwargs.get('num_filters', None))) use_bias = kwargs.get('bias', use_bias) padding_mode = padding_mode.lower().replace('zeros', 'zero') if isinstance(padding_mode, str) else padding_mode if isinstance(padding, str) and auto_pad==False: auto_pad = (padding.lower() == 'same') elif isinstance(padding, int): padding = _pair(padding) elif isinstance(padding, tuple): pass groups=1 super(DepthwiseConv2d, self).__init__(rank, kernel_size, num_filters, strides, auto_pad, padding, padding_mode, use_bias, dilation, groups, transposed=False, name=name, depth_multiplier=depth_multiplier, depthwise=True, separable=False, **kwargs) self.activation = get_activation(activation)
[docs] def conv2d_forward(self, x): self.rank = 2 if len(self.padding)==self.rank: self.padding=(self.padding[1],self.padding[1],self.padding[0],self.padding[0]) if self.padding_mode == 'circular': expanded_padding = ( (self.padding[0] + 1) // 2, self.padding[1] // 2, (self.padding[2] + 1) // 2, self.padding[3] // 2) x = F.pad(x, expanded_padding, mode='circular') else: x = F.pad(x, self.padding, mode='constant' if self.padding_mode == 'zero' else self.padding_mode) return F.conv2d(x, self.weight, self.bias, self.strides, _pair(0), self.dilation, self.groups)
[docs] def forward(self, *x): x = enforce_singleton(x) x = self.conv2d_forward(x) if self.activation is not None: x = self.activation(x) return x
[docs]class DepthwiseConv3d(_ConvNd): def __init__(self, kernel_size, depth_multiplier=1, strides=1, auto_pad=True, padding=None,padding_mode='zero', activation=None, use_bias=False, dilation=1, groups=1, name=None, **kwargs): rank = 3 kernel_size = _triple(kernel_size) strides = _triple(kwargs.get('stride', strides)) dilation = _triple(kwargs.get('dilation_rate', dilation)) num_filters =kwargs.get('filters', kwargs.get('out_channels', kwargs.get('num_filters', None))) use_bias = kwargs.get('bias', use_bias) padding_mode = padding_mode.lower().replace('zeros', 'zero') if isinstance(padding_mode, str) else padding_mode if isinstance(padding, str) and auto_pad==False: auto_pad = (padding.lower() == 'same') elif isinstance(padding, int): padding = _triple(padding) elif isinstance(padding, tuple): pass super(DepthwiseConv3d, self).__init__(rank, kernel_size, num_filters, strides, auto_pad, padding, padding_mode, use_bias, dilation, groups, transposed=False, name=name, depth_multiplier=depth_multiplier, depthwise=False, separable=False, **kwargs) self.activation = get_activation(activation) self._built = False
[docs] def forward(self, *x): x = enforce_singleton(x) if self.padding_mode == 'circular': expanded_padding = ((self.padding[2] + 1) // 2, self.padding[2] // 2, (self.padding[1] + 1) // 2, self.padding[1] // 2,(self.padding[0] + 1) // 2, self.padding[0] // 2) x = F.pad(x, expanded_padding, mode='circular') else: x = F.pad(x, (self.padding[2], self.padding[2], self.padding[1], self.padding[1], self.padding[0], self.padding[0]), mode='constant' if self.padding_mode == 'zero' else self.padding_mode) x = self.conv1(x) if self.activation is not None: x = self.activation(x) return x
# class MixConv2d(Layer): # MixConv: Mixed Depthwise Convolutional Kernels https://arxiv.org/abs/1907.09595 # def __init__(self, in_ch, out_ch, k=(3, 5, 7), stride=1, dilation=1, bias=True, method='equal_params'): # super(MixConv2d, self).__init__() # # groups = len(k) # if method == 'equal_ch': # equal channels per group # i = torch.linspace(0, groups - 1E-6, out_ch).floor() # out_ch indices # ch = [(i == g).sum() for g in range(groups)] # else: # 'equal_params': equal parameter count per group # b = [out_ch] + [0] * groups # a = np.eye(groups + 1, groups, k=-1) # a -= np.roll(a, 1, axis=1) # a *= np.array(k) ** 2 # a[0] = 1 # ch = np.linalg.lstsq(a, b, rcond=None)[0].round().astype(int) # solve for equal weight indices, ax = b # # self.m = nn.ModuleList([nn.Conv2d(in_channels=in_ch, # out_channels=ch[g], # kernel_size=k[g], # stride=stride, # padding=k[g] // 2, # 'same' pad # dilation=dilation, # bias=bias) for g in range(groups)]) # # def forward(self, x): # return torch.cat([m(x) for m in self.m], 1) class DeformConv2d(Layer): def __init__(self, kernel_size, num_filters=None, strides=1, offset_group=2, auto_pad=True, padding_mode='zero', activation=None, use_bias=False, dilation=1, groups=1, name=None, depth_multiplier=None, **kwargs): super(DeformConv2d, self).__init__() self.rank=2 self.kernel_size = _pair(kernel_size) self.num_filters = kwargs.get('num_filters') if self.num_filters is None and depth_multiplier is not None: self.depth_multiplier = depth_multiplier self.dilation = _pair(dilation) self.strides = _pair(strides) self.use_bias = use_bias self.auto_pad = auto_pad self.padding_mode = padding_mode self.activation = get_activation(activation) self.padding = kwargs.get('padding', None) if self.padding is not None and isinstance(self.padding, int): if self.padding > 0: self.auto_pad = False self.padding = _pair(self.padding) else: self.padding = _pair(0) self.groups = groups if self.input_filters % self.groups != 0: raise ValueError('in_channels must be divisible by groups') def build(self, input_shape): if self._built == False: if self.num_filters % self.groups != 0: raise ValueError('out_channels must be divisible by groups') self.offset = Parameter(torch.Tensor(self.input_filters, 2 * self.input_filters, 3, 3)) init.kaiming_uniform_(self.offset, a=math.sqrt(5)) self.weight = Parameter( torch.Tensor(self.num_filters, self.input_filters // self.groups, *self.kernel_size)) init.kaiming_uniform_(self.weight, a=math.sqrt(5)) if self.use_bias: self.bias = Parameter(torch.empty(self.num_filters)) init.zeros_(self.bias) self._built = True def offetconv2d_forward(self, x): if self.auto_pad == True: ih, iw = list(x.size())[-2:] kh, kw = self.kernel_size[-2:] sh, sw = self.strides[-2:] dh, dw = self.dilation[-2:] oh, ow = math.ceil(ih / sh), math.ceil(iw / sw) pad_h = max((oh - 1) * sh + (kh - 1) * dh + 1 - ih, 0) pad_w = max((ow - 1) * sw + (kw - 1) * dw + 1 - iw, 0) if pad_h > 0 or pad_w > 0: x = F.pad(x, [pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2], mode=self.padding_mode) return F.conv2d(x, self.offset, None, (1, 1), (0, 0), (1, 1), (1, 1)) def conv2d_forward(self, x): if self.auto_pad == True: ih, iw = list(x.size())[-2:] kh, kw = self.kernel_size[-2:] sh, sw = self.strides[-2:] dh, dw = self.dilation[-2:] oh, ow = math.ceil(ih / sh), math.ceil(iw / sw) pad_h = max((oh - 1) * sh + (kh - 1) * dh + 1 - ih, 0) pad_w = max((ow - 1) * sw + (kw - 1) * dw + 1 - iw, 0) if pad_h > 0 or pad_w > 0: x = F.pad(x, [pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2], mode='constant' if self.padding_mode=='zero' else self.padding_mode) return F.conv2d(x, self.weight, self.bias, self.strides, self.padding, self.dilation, self.groups) def forward(self, *x): """ Arguments: input (Tensor[batch_size, in_channels, in_height, in_width]): input tensor offset (Tensor[batch_size, 2 * offset_groups * kernel_height * kernel_width, out_height, out_width]): offsets to be applied for each position in the convolution kernel. """ x = enforce_singleton(x) # B 2*input,H,W offset = self.offetconv2d_forward(x).round_() # 2,H,W-->B,2,H,W grid = meshgrid(x.shape[3], x.shape[2]).unsqueeze(0).repeat(x.size(0)) offset = grid + offset deform_x = x.view(x.size(0), x.size(1), x.size(2) * x.size(3)) def __repr__(self): s = self.__class__.__name__ + '(' s += '{in_channels}' s += ', {out_channels}' s += ', kernel_size={kernel_size}' s += ', stride={stride}' s += ', padding={padding}' if self.padding != (0, 0) else '' s += ', dilation={dilation}' if self.dilation != (1, 1) else '' s += ', groups={groups}' if self.groups != 1 else '' s += ', bias=False' if self.bias is None else '' s += ')' return s.format(**self.__dict__) class GcdConv1d(Layer): def __init__(self, kernel_size, num_filters=None, strides=1, auto_pad=True, padding_mode='zero', activation=None, use_bias=False, dilation=1, divisor_rank=0, self_norm=True, is_shuffle=False, name=None, depth_multiplier=None, **kwargs): super(GcdConv1d, self).__init__() self.rank=1 self.kernel_size = _single(kernel_size) self.num_filters = num_filters if self.num_filters is None and depth_multiplier is not None: self.depth_multiplier = depth_multiplier self.strides = _single(strides) self.auto_pad = auto_pad self.padding = 0 self.padding_mode = padding_mode self.activation = get_activation(activation) self.dilation = _single(dilation) self.self_norm = self_norm self.norm = None self.is_shuffle = is_shuffle self.use_bias = use_bias self.divisor_rank = divisor_rank self.crossgroup_fusion = False self.weight = None self.bias = None self.groups = 1 self._built = False def calculate_gcd(self): if self.input_filters is None or not isinstance(self.input_filters, int): raise ValueError('in_channels must be integer ') gcd_list = gcd(self.input_filters, self.num_filters) if len(gcd_list) == 0: self.groups = self.input_filters self.num_filters_1 = self.input_filters else: self.gcd = gcd_list[0] self.groups = gcd_list[min(int(self.divisor_rank), len(gcd_list))] if self.input_filters == self.num_filters or self.input_filters == self.gcd or self.num_filters == self.gcd: self.groups = gcd_list[min(int(self.divisor_rank + 1), len(gcd_list))] def build(self, input_shape): if self._built == False: self.calculate_gcd() print('input:{0} -> output:{1} {2} {3} gcd:{4} group:{5} 通道縮放倍數:{5} '.format(self.input_filters, self.num_filters, self.input_filters // self.groups, self.num_filters // self.groups, self.gcd, self.groups, self.num_filters / self.num_filters)) self.channel_kernal = 2 if self.crossgroup_fusion == True and self.groups > 3 else 1 self.channel_dilation = 1 if self.crossgroup_fusion == True and self.groups >= 4 : self.channel_dilation = 2 self.kernel_size = (self.channel_kernal,) + _pair(self.kernel_size) self.dilation = (self.channel_dilation,) + _pair(self.dilation) self.strides = (1,) + _pair(self.strides) reshape_input_shape = [-1, self._input_shape[0] // self.groups, self.groups, self._input_shape[1]] self.weight = Parameter(torch.Tensor(self.num_filters // self.groups, self._input_shape[0] // self.groups, *self.kernel_size)) # init.kaiming_uniform_(self.weight, mode='fan_in') self._parameters['weight'] = self.weight if self.use_bias: self.bias = Parameter(torch.Tensor(self.num_filters // self.groups)) init.zeros_(self.bias) self._parameters['bias'] = self.bias if self.self_norm == True: self.norm = get_normalization('batch') init.ones_(self.norm.weight) init.zeros_(self.norm.bias) self.to(self.device) self._built = True def forward(self, *x): x = enforce_singleton(x) if self.auto_pad: ih, iw = x.size()[-2:] kh, kw = self.kernel_size[-2:] sh, sw = self.strides[-2:] dh, dw = _pair(self.dilation)[-2:] oh, ow = math.ceil(ih / sh), math.ceil(iw / sw) pad_h = max((oh - 1) * sh + (kh - 1) * dh + 1 - ih, 0) pad_w = max((ow - 1) * sw + (kw - 1) * dw + 1 - iw, 0) if pad_h > 0 or pad_w > 0: x = F.pad(x, [pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2], mode=self.padding_mode) x = x.view(x.size(0), x.size(1) // self.groups, self.groups, x.size(2)) pad_g = max((self.groups - 1) * 1 + (self.channel_kernal - 1) * self.channel_dilation + 1 - self.groups, 0) x = F.pad(x, [0, 0, 0, 0, pad_g // 2, pad_g - pad_g // 2], mode='reflect') x = F.conv2d(x, self.weight, self.bias, self.strides, self.padding, self.dilation, 1) if self.is_shuffle == True: x = x.transpose([2, 1]) x = x.view(x.size(0), x.size(1) * x.size(2), x.size(3)) if self.self_norm == True: x = self.norm(x) if self.activation is not None: x = self.activation(x) return x def extra_repr(self): s = 'kernel_size={kernel_size}, {num_filters},strides={strides}' if 'activation' in self.__dict__ and self.__dict__['activation'] is not None: if inspect.isfunction(self.__dict__['activation']): s += ', activation={0}'.format(self.__dict__['activation'].__name__) elif isinstance(self.__dict__['activation'], nn.Module): s += ', activation={0}'.format(self.__dict__['activation']).__repr__() s += ',auto_pad={auto_pad},use_bias={use_bias} ,dilation={dilation}}' if self.gcd != 1: s += ', gcd={gcd},divisor_rank={divisor_rank},self_norm={self_norm},crossgroup_fusion={' \ 'crossgroup_fusion},is_shuffle={is_shuffle} ' if self._input_shape is not None: s += ', input_shape={0}, input_filter={1}'.format(self._input_shape.clone().tolist(), self.input_filters) if self.output_shape is not None: s += ', output_shape={0}'.format(self.output_shape if isinstance(self.output_shape, ( list, tuple)) else self.output_shape.clone().tolist()) # if self.bias is None: # s += ', use_bias=False' return s.format(**self.__dict__)
[docs]class GcdConv2d(_ConvNd): def __init__(self, kernel_size, num_filters=None, strides=1, auto_pad=True, padding_mode='zero', activation=None, use_bias=False, dilation=1, divisor_rank=0, self_norm=True, is_shuffle=False, crossgroup_fusion=True, name=None, depth_multiplier=None, **kwargs): rank = 2 kernel_size = _pair(kernel_size) strides = _pair(kwargs.get('stride', strides)) dilation = _pair(kwargs.get('dilation_rate', dilation)) num_filters = kwargs.get('filters', kwargs.get('out_channels', kwargs.get('num_filters', None))) use_bias = kwargs.get('bias', use_bias) padding_mode = padding_mode.lower().replace('zeros', 'zero') if isinstance(padding_mode, str) else padding_mode padding = kwargs.get('padding', None) if isinstance(padding, str) and auto_pad==False: auto_pad = (padding.lower() == 'same') elif isinstance(padding, int): padding = _pair(padding) elif isinstance(padding, tuple): pass groups = 1 super(GcdConv2d, self).__init__(rank, kernel_size, num_filters, strides, auto_pad, padding, padding_mode, use_bias, dilation, groups, transposed=False, name=name, depth_multiplier=depth_multiplier, depthwise=True, separable=False, **kwargs) self.activation = get_activation(activation) self.norm = None if self_norm == True: self.self_norm = self_norm self.norm = get_normalization('instance') self.is_shuffle = is_shuffle self.divisor_rank = divisor_rank self.crossgroup_fusion = crossgroup_fusion
[docs] def calculate_gcd(self): if self.input_filters is None or not isinstance(self.input_filters, int): raise ValueError('in_channels must be integer ') self.register_buffer('gcd', torch.zeros((1))) gcd_list = gcd(self.input_filters, self.num_filters) if len(gcd_list) == 0: self.groups = self.input_filters self.num_filters_1 = self.input_filters else: self.gcd = torch.tensor(gcd_list[0]) self.groups = gcd_list[min(int(self.divisor_rank), len(gcd_list))] if self.input_filters == self.num_filters or self.input_filters == self.gcd or self.num_filters == self.gcd: self.groups = gcd_list[min(int(self.divisor_rank + 1), len(gcd_list))]
[docs] def get_padding(self, input_shape): pad_w = 0 pad_h = 0 pad_z = 0 if self.auto_pad == True: iz, ih, iw = list(input_shape)[-3:] kz, kh, kw = self.actual_kernel_size[-3:] sz, sh, sw = self.actual_strides[-3:] dz, dh, dw = self.actual_dilation[-3:] oz, oh, ow = math.ceil(iz / sz), math.ceil(ih / sh), math.ceil(iw / sw) pad_z = max((oz - 1) * sz + (kz - 1) * dz + 1 - iz, 0) pad_h = max((oh - 1) * sh + (kh - 1) * dh + 1 - ih, 0) pad_w = max((ow - 1) * sw + (kw - 1) * dw + 1 - iw, 0) if pad_h % 2 == 1 and sh > 1: pad_h += 1 if pad_w % 2 == 1 and sw > 1: pad_w += 1 elif len(self.padding) == 3: pad_z = self.padding[0] pad_h = self.padding[1] * 2 pad_w = self.padding[2] * 2 if self.padding_mode == 'circular': self.padding = (pad_z , 0, pad_h // 2, pad_h - (pad_h // 2), pad_w // 2, pad_w - (pad_w // 2)) else: self.padding = (pad_z , pad_h // 2, pad_w // 2)
[docs] def build(self, input_shape): if self._built == False: self.input_filters = input_shape[0].item() self.calculate_gcd() if self.num_filters is None and self.depth_multiplier is not None: self.num_filters = int(round(self.input_filters * self.depth_multiplier, 0)) if self.input_filters % self.groups != 0: raise ValueError('in_channels must be divisible by groups') print('{0} input:{1} -> output:{2} {3} {4} gcd:{5} group:{6} 通道縮放倍數:{7} '.format(self.name, self.input_filters, self.num_filters, self.input_filters // self.groups, self.num_filters // self.groups, self.gcd, self.groups, self.num_filters / float( self.input_filters))) self.channel_kernal = torch.tensor(2 if self.crossgroup_fusion == True and self.groups > 4 else 1) self.channel_dilation = torch.tensor(1) # if self.crossgroup_fusion == True and self.groups > 6: # self.channel_dilation = torch.tensor(2) self.actual_kernel_size = (self.channel_kernal,) + _pair(self.kernel_size) self.actual_dilation = (self.channel_dilation,) + _pair(self.dilation) self.actual_strides = (1,) + _pair(self.strides) self.get_padding([input_shape[0]//self.groups, self.groups, input_shape[1], input_shape[2]]) self.weight = Parameter(torch.Tensor(self.num_filters // self.groups, self.input_filters // self.groups, *self.actual_kernel_size)) # init.kaiming_uniform_(self.weight, mode='fan_in') if self.use_bias: self.bias = Parameter(torch.Tensor(self.num_filters // self.groups)) init.zeros_(self.bias) else: self.register_parameter('bias',None) self.to(self.device) self._built = True
[docs] def forward(self, *x): x = enforce_singleton(x) x=x.view(x.size(0), x.size(1) // self.groups, self.groups, x.size(2), x.size(3)) x = F.pad(x, (self.padding[2], self.padding[2], self.padding[1], self.padding[1],0,0),mode='constant' if self.padding_mode == 'zero' else self.padding_mode) if self.channel_kernal.item() == 2: x=torch.cat([x,x[:,:,0:1,:,:]],dim=2) x = F.conv3d(x, self.weight, self.bias, self.actual_strides, _triple(0), self.actual_dilation, 1) if self.is_shuffle == True: x = x.transpose_(2, 1) x = x.view(x.size(0), x.size(1) * x.size(2), x.size(3), x.size(4)) if self.self_norm == True: x = self.norm(x) if self.activation is not None: x = self.activation(x) return x
[docs] def extra_repr(self): s = 'kernel_size={kernel_size}, {num_filters},strides={strides}' if 'activation' in self.__dict__ and self.__dict__['activation'] is not None: if inspect.isfunction(self.__dict__['activation']): s += ', activation={0}'.format(self.__dict__['activation'].__name__) elif isinstance(self.__dict__['activation'], nn.Module): s += ', activation={0}'.format(self.__dict__['activation'].__repr__()) s += ',auto_pad={auto_pad},use_bias={use_bias} ,dilation={dilation}' if self.gcd != 1: s += ', divisor_rank={divisor_rank},self_norm={self_norm},crossgroup_fusion={crossgroup_fusion},is_shuffle={is_shuffle} ' # if self._input_shape is not None: # s += ', input_shape={0}, input_filter={1}'.format(self._input_shape.clone().tolist(), # self.input_filters) # if self.output_shape is not None: # s += ', output_shape={0}'.format(self.output_shape if isinstance(self.output_shape, ( # list, tuple)) else self.output_shape.clone().tolist()) # if self.bias is None: # s += ', use_bias=False' return s.format(**self.__dict__)
[docs]class Lambda(Layer): """ Applies a lambda function on forward() Args: function (fn): the lambda function """ def __init__(self, function, name=None): super(Lambda, self).__init__(name=name) self.function = function
[docs] def forward(self, *x): return self.function(*x)
[docs]class Reshape(Layer): """ Reshape the input volume Args: *shape (ints): new shape, WITHOUT specifying batch size as first dimension, as it will remain unchanged. """ def __init__(self, target_shape, name=None): super(Reshape, self).__init__(name=name) if isinstance(target_shape, tuple): self.target_shape = to_tensor([target_shape[i] for i in range(len(target_shape))]) elif isinstance(target_shape, list): self.target_shape = to_tensor(target_shape)
[docs] def forward(self, *x): x = enforce_singleton(x) shp = self.target_shape.tolist().copy() shp.insert(0, x.size(0)) shp = tuple(shp) return torch.reshape(x, shp)
[docs]class SelfAttention(Layer): """ Self attention Layer""" def __init__(self, reduction_factor=8, name=None): super(SelfAttention, self).__init__(name=name) self.rank=2 # self.activation = activation self.reduction_factor = reduction_factor self.query_conv = None self.key_conv = None self.value_conv = None self.attention = None self.gamma = nn.Parameter(torch.zeros(1)) init.zeros_(self.gamma) self._parameters['gamma'] = self.gamma self.softmax = nn.Softmax(dim=-1) #
[docs] def build(self, input_shape): self.query_conv = nn.Conv2d(in_channels=self.input_filters, out_channels=self.input_filters // self.reduction_factor, kernel_size=1) self.key_conv = nn.Conv2d(in_channels=self.input_filters, out_channels=self.input_filters // self.reduction_factor, kernel_size=1) self.value_conv = nn.Conv2d(in_channels=self.input_filters, out_channels=self.input_filters, kernel_size=1) self.to(self.device)
[docs] def forward(self, *x): """ inputs : x : input feature maps( B X C X W X H) returns : out : self attention value + input feature attention: B X N X N (N is Width*Height) """ x = enforce_singleton(x) B, C, width, height = x.size() proj_query = self.query_conv(x).view(B, -1, width * height).permute(0, 2, 1) # B X CX(N) proj_key = self.key_conv(x).view(B, -1, width * height) # B X C x (*W*H) energy = torch.bmm(proj_query, proj_key) # transpose check self.attention = self.softmax(energy).clone() # BX (N) X (N) proj_value = self.value_conv(x).view(B, -1, width * height) # B X C X N out = torch.bmm(proj_value, self.attention.permute(0, 2, 1)) out = out.view(B, C, width, height) out = self.gamma * out.clone() + x return out
""" Implementation of the CoordConv modules from https://arxiv.org/abs/1807.03247 """ def _append_coords(input_tensor, with_r=False): batch_size, _, x_dim, y_dim = input_tensor.size() xx_channel = torch.arange(x_dim).repeat(1, y_dim, 1) yy_channel = torch.arange(y_dim).repeat(1, x_dim, 1).transpose(1, 2) xx_channel = xx_channel.float() / (x_dim - 1) yy_channel = yy_channel.float() / (y_dim - 1) xx_channel = xx_channel * 2 - 1 yy_channel = yy_channel * 2 - 1 xx_channel = xx_channel.repeat(batch_size, 1, 1, 1).transpose(2, 3) yy_channel = yy_channel.repeat(batch_size, 1, 1, 1).transpose(2, 3) ret = torch.cat([input_tensor, xx_channel.type_as(input_tensor), yy_channel.type_as(input_tensor), ], dim=1, ) if with_r: rr = torch.sqrt( torch.pow(xx_channel.type_as(input_tensor) - 0.5, 2) + torch.pow(yy_channel.type_as(input_tensor) - 0.5, 2)) ret = torch.cat([ret, rr], dim=1) return ret """ An alternative implementation for PyTorch with auto-infering the x-y dimensions. https://github.com/mkocabas/CoordConv-pytorch/blob/master/CoordConv.py """
[docs]class CoordConv2d(Layer): def __init__(self, kernel_size, num_filters, strides, auto_pad=True, activation=None, use_bias=False, group=1, dilation=1, with_r=False, name=None, **kwargs): super().__init__(name=name) self.rank=2 self.kernel_size = kernel_size self.num_filters = num_filters self.strides = strides self.auto_pad = auto_pad self.use_bias = use_bias self.group = group self.dilation = dilation self._conv_settings = kwargs self.activation = get_activation(activation) self.addcoords = partial(_append_coords, with_r=with_r) self.conv = None
[docs] def build(self, input_shape): if self._built == False: self.conv = Conv2d(self.kernel_size, self.num_filters, self.strides, auto_pad=self.auto_pad, activation=self.activation, use_bias=self.use_bias, group=self.group, dilation=self.dilation, **self._conv_settings) self._built = True
[docs] def forward(self, *x): x = enforce_singleton(x) ret = self.addcoords(x) ret = self.conv(ret) return ret
[docs]class Upsampling2d(Layer): def __init__(self, size=None, scale_factor=None, mode='nearest', align_corners=True, name=None): super(Upsampling2d, self).__init__(name=name) self.rank=2 self.size = size if isinstance(scale_factor, tuple): self.scale_factor = tuple(float(factor) for factor in scale_factor) else: self.scale_factor = float(scale_factor) if scale_factor else None self.mode = mode self.align_corners = align_corners
[docs] def forward(self, *x): x = enforce_singleton(x) if self.mode == 'pixel_shuffle': return F.pixel_shuffle(x, int(self.scale_factor)) elif self.mode == 'nearest': return F.interpolate(x, self.size, self.scale_factor, self.mode, None) else: return F.interpolate(x, self.size, self.scale_factor, self.mode, self.align_corners)
[docs] def extra_repr(self): if self.scale_factor is not None: info = 'scale_factor=' + str(self.scale_factor) else: info = 'size=' + str(self.size) info += ', mode=' + self.mode return info
[docs]class Dropout(Layer): def __init__(self, dropout_rate=0, name=None): super(Dropout, self).__init__(name=name) self.inplace = True if dropout_rate < 0 or dropout_rate > 1: raise ValueError("dropout probability has to be between 0 and 1, ""but got {}".format(dropout_rate)) self.dropout_rate = dropout_rate
[docs] def forward(self, *x): x = enforce_singleton(x) return F.dropout(x, self.dropout_rate, self.training, self.inplace)
[docs] def extra_repr(self): return 'p={}, inplace={}'.format(self.dropout_rate, self.inplace)
[docs]class AlphaDropout(Layer): ''' .. _Self-Normalizing Neural Networks: https://arxiv.org/abs/1706.02515 ''' def __init__(self, dropout_rate=0, name=None): super(AlphaDropout, self).__init__(name=name) self.inplace = True if dropout_rate < 0 or dropout_rate > 1: raise ValueError("dropout probability has to be between 0 and 1, ""but got {}".format(dropout_rate)) self.dropout_rate = dropout_rate
[docs] def forward(self, *x): x = enforce_singleton(x) return F.alpha_dropout(x, self.dropout_rate, self.training, self.inplace)
[docs] def extra_repr(self): return 'p={}, inplace={}'.format(self.dropout_rate, self.inplace)
[docs]class SingleImageLayer(Layer): def __init__(self, image,is_recursive=False,name=None): super(SingleImageLayer, self).__init__(name=name) self.rank=2 if isinstance(image,(np.ndarray,torch.Tensor)): self.origin_image = to_tensor(image).squeeze() self.input_shape = image.shape[1:]
[docs] def build(self, input_shape): if self._built == False: self.weight = Parameter(self.origin_image.clone(), requires_grad=True) self.input_filters = input_shape[0] self._built = True
[docs] def forward(self,x): return self.weight.data.unsqueeze(0)
[docs] def extra_repr(self): return 'is_recursive={0}'.format(self.is_recursive)