nonlinear 0.0.14

A hand-curated collections of activations functions for deep learning research.

Non-Linear

A hand-curated collections of activations functions for deep learning research.

Channel-Based Activation Functions

ID	Activation Function	Formula
1	DPReLU	$f(x)=ax$ if $x\ge0$, else $bx$
2	DualLine	$f(x)=ax+m$ if $x\ge0$, else $bx+m$
3	EPReLU	$f(x)=kx$ if $x\ge0$, else $\dfrac{x}{a}$
4	FPAF	$f(x)=a\mu(x)$ if $x\ge0$, else $b\nu(x)$
5	FReLU	$f(x)=\mathrm{ReLU}(x)+b$
6	LearnableTeLU	$f(x)=x\tanh(\mathrm{ELU}(ax))$
7	LeLeLU	$f(x)=ax$ if $x\ge0$, else $0.01ax$
8	PairedReLU	$f(x)=\left[\mathrm{ReLU}(ax-b),\mathrm{ReLU}(cx-d)\right]$ (concat on channels)
9	PiLU	$f(x)=ax+c(1-a)$ if $x\ge c$, else $bx+c(1-b)$
10	PREU	$f(x)=ax$ if $x\ge0$, else $ax,e^{bx}$
11	PTELU	$f(x)=x$ if $x\ge0$, else $\lvert a\rvert,\tanh(\lvert b\rvert x)$
12	RMAF	$f(x)=\dfrac{abx}{\left(0.25(1+e^{-x})+0.75\right)^c}$
13	RTPReLU	$f(x)=x$ if $x+\eta\ge0$, else $x/a$; $\eta\sim\mathcal{N}(0,\sigma^2)$ during training
14	ShiLU	$f(x)=a,\mathrm{ReLU}(x)+b$
15	StarReLU	$f(x)=a,\mathrm{ReLU}(x)^2+b$
16	TaLU	$f(x)=x$ if $x\ge\lvert b\rvert$; $\tanh(x)$ if $x>\lvert a\rvert$; else $\tanh(\lvert a\rvert)$
17	TanhLU	$f(x)=a,\tanh(cx)+bx$

Static Activation Functions

ID	Activation Function	Formula
1	ShiftedReLU	$f(x)=\max(x,-1)$
2	ADA	$f(x)=x$ if $x\ge0$, else $xe^x$
3	OAF	$f(x)=\mathrm{ReLU}(x)+x\sigma(x)$
4	AbsLU	$f(x)=x$ if $x\ge0$, else $\alpha,\mathrm{abs}(x)$
5	ParametricLogish	$f(x)=\alpha x\log(1+\sigma(\beta x))$
6	ExpExpish	$f(x)=xe^{-e^{-x}}$
7	DoubleSiLU	$f(x)=\dfrac{x}{1+\exp!\left(-\dfrac{x}{1+e^{-x}}\right)}$
8	GeneralizedSwish	$f(x)=x,\sigma(e^{-x})$
9	MSiLU	$f(x)=x\sigma(x)+\dfrac{1}{4}e^{-x^2-1}$
10	TBSReLU	$f(x)=x\tanh!\left(\dfrac{1-e^{-x}}{1+e^{-x}}\right)$
11	ASiLU	$f(x)=\arctan!\left(\dfrac{x}{1+e^{-x}}\right)$
12	NoisyReLU	$f(x)=x+\epsilon,\mathrm{std}(x)$ if $x\ge0$, else $0$; $\epsilon\sim\mathcal{N}(0,1)$
14	SaRa	$f(x)=x$ if $x\ge0$, else $\dfrac{x}{1+\alpha e^{-\beta x}}$
15	SiELU	$f(x)=x,\sigma!\left(2\sqrt{2/\pi}(x+0.044715x^3)\right)$
16	EANAF	$f(x)=x\tanh(\mathrm{softplus}(x)/2)$
17	MaxSig	$f(x)=\max(x,\sigma(x))$
18	TangentSigmoidReLU	$f(x)=x\tanh(\sigma(x))$
19	Phish	$f(x)=x\tanh(\mathrm{GELU}(x))$
20	SelfArctan	$f(x)=x\arctan(x)$
21	PFLU	$f(x)=\dfrac{x}{2}\left(1+\dfrac{x}{\sqrt{1+x^2}}\right)$
22	ReSP	$f(x)=\alpha x+\log 2$ if $x\ge0$, else $\log(1+e^x)$
23	Serf	$f(x)=x,\mathrm{erf}(\log(1+e^x))$
24	LogSigmoid	$f(x)=\log(\sigma(x))$
25	SlopedReLU	$f(x)=\alpha x$ if $x\ge0$, else $0$
26	ReCU	$f(x)=\mathrm{ReLU}(x^3)$
27	MinSin	$f(x)=\min(x,\sin x)$
28	LaLU	$f(x)=x(1-0.5e^{-x})$ if $x\ge0$, else $0.5xe^x$
29	mReLU	$f(x)=\min(\mathrm{ReLU}(1-x),\mathrm{ReLU}(1+x))$
30	FlattedTSwish	$f(x)=\mathrm{ReLU}(x)\sigma(x)+t$
31	ERF	$f(x)=x,\mathrm{erf}(\alpha x)$
32	RePU	$f(x)=\mathrm{ReLU}(x^{\alpha})$
33	TangentBipolarSigmoidReLU	$f(x)=x\tanh!\left(\dfrac{1-e^{-x}}{1+e^{-x}}\right)$
35	Logish	$f(x)=x\log(1+\sigma(x))$
36	TripleStateSwish	$f(x)=x\sigma(x)[\sigma(x)+\sigma(x-\alpha)+\sigma(x-\beta)]$
37	ReQU	$f(x)=\mathrm{ReLU}(x^2)$
38	ExponentialSwish	$f(x)=e^{-x}\sigma(x)$
39	SinSig	$f(x)=x\sin!\left(\dfrac{\pi}{2}\sigma(x)\right)$
40	PLAF	$f(x)=x-(1-\dfrac{1}{d})$ if $x\ge1$; $-x-(1-\dfrac{1}{d})$ if $x<-1$; else $\dfrac{1}{d}(\mathrm{abs}(x))^d$
41	TeLU	$f(x)=x\tanh(e^x)$
42	DiffELU	$f(x)=x$ if $x\ge0$, else $a(xe^x-be^{bx})$
43	Elliot	$f(x)=0.5+\dfrac{0.5x}{1+\mathrm{abs}(x)}$
44	SoftModulusQ	$f(x)=x^2(2-\mathrm{abs}(x))$ if $\mathrm{abs}(x)\le1$, else $\mathrm{abs}(x)$
45	DerivativeSiLU	$f(x)=\sigma(x)\left(1+x(1-\sigma(x))\right)$
46	NLReLU	$f(x)=\log(\beta\max(0,x)+1)$
47	IpLU	$f(x)=x$ if $x\ge0$, else $\dfrac{x}{1+(\mathrm{abs}(x))^{\alpha}}$
48	ThLU	$f(x)=x$ if $x\ge0$, else $\tanh(x/2)$
49	RReLU	$f(x)=x$ if $x\ge0$, else $x/a$ with $a\sim U(l,u)$ during training
50	PolyLU	$f(x)=x$ if $x\ge0$, else $\dfrac{1}{1-x}-1$
51	Suish	$f(x)=\max(x,xe^{-\mathrm{abs}(x)})$
52	TSiLU	$\alpha=\dfrac{x}{1+e^{-x}},;f(x)=\dfrac{e^{\alpha}-e^{-\alpha}}{2e^{\alpha}}$
53	SoftsignRReLU	$f(x)=\dfrac{1}{(1+x)^2}+x$ if $x\ge0$, else $\dfrac{1}{(1+x)^2}+ax$ with $a\sim U(l,u)$
54	SoftModulusT	$f(x)=x\tanh(x/\alpha)$
55	Gish	$f(x)=x\log(2-e^{-e^x})$
56	DRLU	$f(x)=\mathrm{ReLU}(x-\alpha)$
57	NReLU	$f(x)=x+\epsilon,\mathrm{std}(x)$ if $x\ge0$, else $0$; $\epsilon\sim\mathcal{N}(0,1)$
58	LogLogish	$f(x)=x(1-e^{-e^x})$
59	SGELU	$f(x)=\alpha x,\mathrm{erf}(x/\sqrt{2})$
60	TSReLU	$f(x)=x\tanh(\sigma(x))$
61	REU	$f(x)=x$ if $x\ge0$, else $xe^x$
62	ReSech	$f(x)=x,\mathrm{sech}(x)=\dfrac{2x}{e^x+e^{-x}}$
63	SineReLU	$f(x)=x$ if $x\ge0$, else $\epsilon(\sin x-\cos x)$
65	DLU	$f(x)=x$ if $x\ge0$, else $\dfrac{x}{1-x}$
66	CaLU	$f(x)=x\left(\dfrac{\arctan(x)}{\pi}+b\right)$
67	RandomizedSlopedReLU	$f(x)=\alpha x$ if $x\ge0$, else $0$; $\alpha\sim U(1,10)$ at init
68	TanhExp	$f(x)=x\tanh(e^x)$
69	PoLU	$f(x)=x$ if $x\ge0$, else $(1-x)^{-\alpha}-1$
70	GCU	$f(x)=x\cos x$
71	SigmoidDerivative	$f(x)=e^{-x}\sigma(x)^2$
72	Smish	$f(x)=x\tanh(\log(1+\sigma(x)))$
73	SigLU	$f(x)=x$ if $x\ge0$, else $\dfrac{1-e^{-2x}}{1+e^{-2x}}$
74	AOAF	$f(x)=\mathrm{ReLU}(x-b\bar{x})+c\bar{x}$, $\bar{x}$ is channel mean over batch/spatial dims