B
|
_c ã @ s d Z ddlZddlmZ ddlmZ ddlmZmZ ddlm Z G dd dej
|
Z G d d
|
d
|
ej
|
Z d d Z d d Zdd ZdddZdS )z9
|
@author: liaoxingyu
|
@contact: sherlockliao01@gmail.com
|
é N)Únn)Ú BatchNorm2d)ÚReLUÚ LeakyReLU)Ú Parameterc s4 e Zd Z fddZdd Zdd Zdd Z ZS ) ÚTLUc s0 t t| ¡ || _tt |¡| _| ¡ dS )z9max(y, tau) = max(y - tau, 0) + tau = ReLU(y - tau) + tauN) Úsuperr Ú__init__Ú num_featuresr ÚtorchÚTensorÚtauÚreset_parameters)Úselfr
|
)Ú __class__© ú./layers/frn.pyr s z TLU.__init__c C s t j | j¡ d S )N)r ÚinitÚzeros_r )r r r r r s zTLU.reset_parametersc C s dj f | jS )Nznum_features={num_features})ÚformatÚ__dict__)r r r r Ú
|
extra_repr s zTLU.extra_reprc C s t || j d| jdd¡¡S )Né )r Úmaxr Úviewr
|
)r Úxr r r Úforward s z TLU.forward)Ú__name__Ú
|
__module__Ú __qualname__r r r r Ú __classcell__r r )r r r s r c s6 e Zd Zd fdd Zdd Zdd Zd d
|
Z ZS ) ÚFRNçíµ ÷Æ°>Fc sv t t| ¡ || _|| _|| _tt |¡| _ tt |¡| _
|
|rVtt d¡| _ n| dt |g¡¡ | ¡ dS )zë
|
weight = gamma, bias = beta
|
beta, gamma:
|
Variables of shape [1, 1, 1, C]. if TensorFlow
|
Variables of shape [1, C, 1, 1]. if PyTorch
|
eps: A scalar constant or learnable variable.
|
r ÚepsN)r r! r r
|
Úinit_epsÚis_eps_leanabler r r ÚweightÚbiasr# Úregister_bufferr )r r
|
r# r% )r r r r ! s z FRN.__init__c C s8 t j | j¡ t j | j¡ | jr4t j | j| j ¡ d S )N)
|
r r Úones_r& r r' r% Ú constant_r# r$ )r r r r r 7 s zFRN.reset_parametersc C s dj f | jS )Nz+num_features={num_features}, eps={init_eps})r r )r r r r r = s zFRN.extra_reprc C s` | d¡jddgdd}|t || j ¡ ¡ }| j d| jdd¡| | j d| jdd¡ }|S )ai
|
0, 1, 2, 3 -> (B, H, W, C) in TensorFlow
|
0, 1, 2, 3 -> (B, C, H, W) in PyTorch
|
TensorFlow code
|
nu2 = tf.reduce_mean(tf.square(x), axis=[1, 2], keepdims=True)
|
x = x * tf.rsqrt(nu2 + tf.abs(eps))
|
# This Code include TLU function max(y, tau)
|
return tf.maximum(gamma * x + beta, tau)
|
é é T)ÚdimÚkeepdimr )
|
ÚpowÚmeanr Úrsqrtr# Úabsr& r r
|
r' )r r Znu2r r r r @ s ,z FRN.forward)r" F)r r r r r r r r r r )r r r! s r! c C s | }d}d}d}x| ¡ D ]x\}}|rpt|ttfrpt|trT| |t|jd¡ nt | |t |jd¡ n| |t
|
|¡ |}|}t|t}qW |S )z5
|
Convert 'BatchNorm2d + ReLU' to 'FRN + TLU'
|
NF)r
|
) Únamed_childrenÚ
|
isinstancer r r Ú
|
add_moduler! r
|
ÚNotImplementedErrorr Ú bnrelu_to_frn)ÚmoduleÚmodZ before_nameZ before_childÚ is_before_bnÚnameÚchildr r r r7 V s
|
r7 c C s | }d }x| ¡ D ]t\}}t||rtt||rtt|trR|j}| |t|jd¡ t|tt fr| |t
|
|d¡ q| |t ||¡ qW |S )N)r
|
) r3 ÚhasattrÚgetattrr4 r r
|
r5 r! r r r Úconvert)r8 Ú flag_namer9 Z before_chr; r< r r r r? s s
|
r? c C sX | }xN| ¡ D ]B\}}t|dr>t|| | |t||¡ q| |t||¡ qW |S )NÚis_convert_frn)r3 r= Údelattrr5 Ú remove_flags)r8 r@ r9 r; r< r r r rC s
|
|
rC ©r, é rE r+ rA c sÎ t t dgfdd}dg fdd}t|trL|g}fdd|D } |¡ |¡ | }| ¡ ¡ xD ] }| ¡ qW x D ] }| ¡ q¢W tdtdS ) NFc s& fdd} |¡¡ d S )Nc sb t tjtjfskr( d¡ d S d o<t | ttf}|rNt| d t | t¡ d S )NFéÿÿÿÿT) r4 r Ú
|
SequentialÚ
|
ModuleListÚappendr r Úsetattrr )r ÚinputÚoutputÚ is_converted)r@ r: Úmodelr8 r r Úhook s
|
z;bnrelu_to_frn2.<locals>.register_forward_hook.<locals>.hook)rI Úregister_forward_hook)r8 rO )r@ Ú forard_hooksr: rN )r8 r rP s z-bnrelu_to_frn2.<locals>.register_forward_hookc s& fdd} |¡¡ d S )Nc sb t tjtjfskr( d¡ d S d o8t | t}|rJt| d t | ttf¡ d S )NFrF T) r4 r rG rH rI r rJ r r )r rK rL rM )r@ Úis_before_relurN r8 r r rO ¥ s
|
z<bnrelu_to_frn2.<locals>.register_backward_hook.<locals>.hook)rI Úregister_backward_hook)r8 rO )Úbackward_hooksr@ rR rN )r8 r rS ¤ s z.bnrelu_to_frn2.<locals>.register_backward_hookc s g | ]}t j f| qS r )r Úrand)Ú.0Zin_size)Ú
|
batch_sizer r ú
|
<listcomp>µ s z"bnrelu_to_frn2.<locals>.<listcomp>)r@ ) Úlistr4 ÚtupleÚapplyÚsumÚbackwardÚremover? rC ) rN Ú
|
input_sizerW r@ rP rS r rL Úhr )rT rW r@ rQ r: rR rN r Úbnrelu_to_frn2 s(
|
|
|
|
|
ra )rD r+ rA )Ú__doc__r r Ztorch.nn.modules.batchnormr Útorch.nnr r Ztorch.nn.parameterr ÚModuler r! r7 r? rC ra r r r r Ú<module> s 6
|
