گرادیان کاهشی تصادفی

${\displaystyle \mathcal{J}(\mathit{\theta })=\left(\frac{1}{n}\right)\sum _{\mathit{i}=1}^n{\displaystyle {\mathcal{J}}_{{\displaystyle i}}(\mathit{\theta })}}$

${\displaystyle \mathcal{J}(\mathit{\theta })=E[{\displaystyle {\mathcal{J}}_{{\displaystyle i}}(\mathit{\theta })]}}$

${\displaystyle \theta =\theta -\alpha {\mathrm{\nabla }}_{\theta }{\mathcal{J}}_{\mathit{i}}(\theta ;x^{(i)},y^{(i)})}$

${\displaystyle \mathrm{\Delta }\theta =\eta \mathrm{\Delta }\theta -\alpha \mathrm{\nabla }\mathcal{J}(\theta )}$

${\displaystyle \theta =\theta +\mathrm{\Delta }\theta }$

${\displaystyle \theta =\theta -\alpha \mathrm{\nabla }{\mathcal{J}}_{\mathcal{i}}(\theta )+\eta \mathrm{\Delta }\theta }$

${\displaystyle G_{j,j}=\sum _{\tau =1}^t{g}_{\tau ,j}^2}$

${\displaystyle \theta =\theta -\eta \mathrm{d}\mathrm{i}\mathrm{a}\mathrm{g}(G{)}^{-\frac{1}{2}}\circ g}$

${\displaystyle {\theta }_j={\theta }_j-\frac{\alpha }{\sqrt{G_{j,j}}}{g}_j.}$

${\displaystyle v(\theta ,t)=\gamma v(\theta ,t-1)+(1-\gamma )(\mathrm{\nabla }{\mathcal{J}}_i(\theta ){)}^2}$

${\displaystyle \theta =\theta -\frac{\alpha }{\sqrt{v(\theta ,t)}}\mathrm{\nabla }{\mathcal{J}}_i(\theta )}$

${\displaystyle m_{\theta }^{(t+1)}\leftarrow {\beta }_1{m}_{\theta }^{(t)}+(1-{\beta }_1){\mathrm{\nabla }}_{\theta }{J}^{(t)}}$

${\displaystyle v_{\theta }^{(t+1)}\leftarrow {\beta }_2{v}_{\theta }^{(t)}+(1-{\beta }_2)({\mathrm{\nabla }}_{\theta }{J}^{(t)}{)}^2}$

${\displaystyle {\hat{m}}_{\theta }=\frac{m_{\theta }^{(t+1)}}{1-({\beta }_1{)}^{t+1}}}$

${\displaystyle {\hat{v}}_{\theta }=\frac{v_{\theta }^{(t+1)}}{1-({\beta }_2{)}^{t+1}}}$

${\displaystyle {\theta }^{(t+1)}\leftarrow {\theta }^{(t)}-\alpha \frac{{\hat{m}}_{\theta }}{\sqrt{{\hat{v}}_{\theta }}+ϵ}}$